GRPC
GRPC in Python
Install grpc for python
GRPC in Python Server
Start From Proto
// Copyright 2015 gRPC authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.helloworld";
option java_outer_classname = "HelloWorldProto";
package cameramap;
// The greeting service definition.
service Cameramap {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
rpc SayGoodBye (HelloRequest) returns (HelloReply) {}
}
// The request message containing the user's name.
message HelloRequest {
string name = 1;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}
Directory
Before you start to generate grpc files, you must set up .proto as you wish to request and response.
├── cameramap
│ └── cameramap.proto
└── cameramap_server
├──
where the directory cameramap put the proto file and we start to program server from cameramap_server.
Recompile Proto
Into directory cameramap_server.
python -m grpc_tools.protoc -I ../cameramap/ --python_out=. --grpc_python_out=. ../cameramap/cameramap.proto
It will generate cameramap_pb2_grpc.py and cameramap_pb2_grpc.py automatically.
Let's see the directory changes
├── cameramap
│ └── cameramap.proto
└── cameramap_server
├── cameramap_pb2_grpc.py
├── cameramap_pb2_grpc.pyc
├── cameramap_pb2.py
├── cameramap_pb2.pyc
└── server.py
server.py
from concurrent import futures
import time
import math
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.name
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.name)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
cameramap_pb2_grpc.add_CameramapServicer_to_server(CameramapServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
Run the python server
GRPC in Python Client
In cameramap_client directory
python -m grpc_tools.protoc -I ../cameramap/ --python_out=. --grpc_python_out=. ../cameramap/cameramap.proto
Let's see the directory first
root@ubuntu:~/grpc# tree
.
├── cameramap
│ └── cameramap.proto
├── cameramap_client
│ ├── cameramap_pb2_grpc.py
│ └── cameramap_pb2.py
└── cameramap_server
├── cameramap_pb2_grpc.py
├── cameramap_pb2_grpc.pyc
├── cameramap_pb2.py
├── cameramap_pb2.pyc
└── server.py
where client.py in directory cameramap_client
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
response = stub.SayHello(cameramap_pb2.HelloRequest(name='you'))
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
Result
root@ubuntu:~/grpc/cameramap_client# python client.py
Greeter client received: Hello, you!
GRPC in Golang
You can prepare the .proto file in directory cameramap as before.
protoc -I cameramap/ cameramap/cameramap.proto --go_out=plugins=grpc:cameramap
Directory Tree
root@golang17:~/golang/projects/wru/src/github.com/jonah/rpcmap# tree
.
├── cameramap
│ ├── cameramap.pb.go
│ └── cameramap.proto
├── cameramap_client
│ └── main.go
├── cameramap_server
│ └── main.go
└── rebuild.sh
It will generate cameramap.pb.go in directory cameramap for your used.
Server Code
package main
import (
"log"
"net"
pb "github.com/jonah/rpcmap/cameramap"
"golang.org/x/net/context"
"google.golang.org/grpc"
//pb "google.golang.org/grpc/examples/helloworld/helloworld"
"fmt"
"google.golang.org/grpc/reflection"
)
const (
port = ":50051"
)
var newpara int
// server is used to implement helloworld.GreeterServer.
type server struct{}
// SayHello implements helloworld.GreeterServer
func (s *server) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) {
fmt.Println(port)
fmt.Println(newpara)
return &pb.HelloReply{Message: "Hello " + in.Name}, nil
}
/* here we put camera map location with k-nn */
func modifypara() {
newpara = 3
}
func main() {
modifypara()
lis, err := net.Listen("tcp", port)
if err != nil {
log.Fatalf("failed to listen: %v", err)
}
s := grpc.NewServer()
// pb.RegisterGreeterServer(s, &server{})
pb.RegisterCameramapServer(s, &server{})
// Register reflection service on gRPC server.
reflection.Register(s)
if err := s.Serve(lis); err != nil {
log.Fatalf("failed to serve: %v", err)
}
}
Client Code
package main
import (
"log"
"os"
"golang.org/x/net/context"
"google.golang.org/grpc"
// pb "google.golang.org/grpc/examples/helloworld/helloworld"
pb "github.com/jonah/rpcmap/cameramap"
//pb "helloworld/helloworld"
)
const (
// address = "localhost:50051"
address = "192.168.51.129:50051"
defaultName = "world"
)
func main() {
// Set up a connection to the server.
conn, err := grpc.Dial(address, grpc.WithInsecure())
if err != nil {
log.Fatalf("did not connect: %v", err)
}
defer conn.Close()
// c := pb.NewGreeterClient(conn)
c := pb.NewCameramapClient(conn)
// Contact the server and print out its response.
name := defaultName
if len(os.Args) > 1 {
name = os.Args[1]
}
r, err := c.SayHello(context.Background(), &pb.HelloRequest{Name: name})
if err != nil {
log.Fatalf("could not greet: %v", err)
}
log.Printf("Greeting: %s", r.Message)
}
Result
You can use one .proto file, and generate it for different programming language used.
How to use Dict in Grpc in Python
.Proto File
where .proto file
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.helloworld";
option java_outer_classname = "HelloWorldProto";
package cameramap;
// The greeting service definition.
service Cameramap {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
rpc SayGoodBye (HelloRequest) returns (HelloReply) {}
}
// The request message containing the user's name.
message HelloRequest {
//string name = 1;
map<string, string> mapfield = 1;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}
where we can see the attribute mapfild is a map type.
Client Code
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
dictt = {'Name': 'Zara', 'Age': '7', 'Class': 'First'}
response = stub.SayHello(cameramap_pb2.HelloRequest(mapfield=dictt))
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
We can directly send a dict to server.
Server Code
from concurrent import futures
import time
import math
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.mapfield
print request.mapfield['Name']
#return cameramap_pb2.HelloReply(message='Hello, %s!' % request.name)
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.mapfield)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
cameramap_pb2_grpc.add_CameramapServicer_to_server(CameramapServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
You can directly parse request.mapfield['Name'] as dict in python.
Note:
You cannot use repeated map in GRPC, since gprc not support it.
Complex Request
We would like to send a complex request composed a list, such as followed
{"a":"b", "c":["1","2","3"], "d":"e"}
Client Code
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
dictt = {'Name': 'Zara', 'Age': '7', 'Class': 'First'}
listt = ["1","2","3"]
# changing list to a string
dictt["list"] = str(listt)
response = stub.SayHello(cameramap_pb2.HelloRequest(mapfield=dictt))
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
Use str() function forces list to a string.
Server Code
from concurrent import futures
import time
import math
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.mapfield
print request.mapfield['Name']
#return cameramap_pb2.HelloReply(message='Hello, %s!' % request.name)
listt = eval(request.mapfield['list'])
# now it become a list
for v in listt:
print v
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.mapfield)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
cameramap_pb2_grpc.add_CameramapServicer_to_server(CameramapServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
Use eval() function to force string to list.
List Requst
message HelloRequest {
//repeated Datatest datatest = 1;
repeated string datatest = 1;
}
Client Code
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
#dictt = {'Name': 'Zara', 'Age': '7', 'Class': 'First'}
listt = ["1","2","3"]
# changing list to a string
#dictt["list"] = str(listt)
response = stub.SayHello(cameramap_pb2.HelloRequest(datatest=listt))
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
Server Code
from concurrent import futures
import time
import math
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.datatest
#return cameramap_pb2.HelloReply(message='Hello, %s!' % request.name)
for v in request.datatest:
print v
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.datatest)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
cameramap_pb2_grpc.add_CameramapServicer_to_server(CameramapServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
Complex Define
In .proto
message Point {
int32 latitude = 1;
int32 longitude = 2;
}
// A latitude-longitude rectangle, represented as two diagonally opposite
// points "lo" and "hi".
message Rectangle {
// One corner of the rectangle.
Point lo = 1;
// The other corner of the rectangle.
Point hi = 2;
}
In Client
def guide_list_features(stub):
rectangle = route_guide_pb2.Rectangle(
lo=route_guide_pb2.Point(latitude=400000000, longitude=-750000000),
hi=route_guide_pb2.Point(latitude=420000000, longitude=-730000000))
print("Looking for features between 40, -75 and 42, -73")
More Complicate Data
It's not allow for grpc to use the folowing request or response.
[{"a":"b"}, {"c":"d"}, {"e":"f"}]
Unfortunately, we usually see the result but it is not strict.
So grpc not allow above requst.
The better way is like this
{"alarms" : [{"a":"b"}, {"c":"d"}, {"e":"f"}] }
We shoud note what the request for, now it's alarms.
So it's still a map but with list inside.
.Proto
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.helloworld";
option java_outer_classname = "HelloWorldProto";
package cameramap;
// The greeting service definition.
service Cameramap {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
rpc SayGoodBye (HelloRequest) returns (HelloReply) {}
}
// The request message containing the user's name.
message HelloRequest {
map<string, string> mapfield = 1;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}
Client Code
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
data =[]
data.append({'Name': 'Zara', 'Age': '7', 'Class': 'First'})
data.append({'Name': 'Zara1', 'Age': '71', 'Class': 'First1'})
dictt = {"data":str(data)}
#a = [{'Name': 'Zara', 'Age': '7', 'Class': 'First'}, {'Name': 'Zara1', 'Age': '71', 'Class': 'First1'}]
# changing list to a string
#dictt["list"] = str(listt)
response = stub.SayHello(cameramap_pb2.HelloRequest(mapfield=dictt))
#response = stub.SayHello(bb)
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
Server Code
from concurrent import futures
import time
import math
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.mapfield
#return cameramap_pb2.HelloReply(message='Hello, %s!' % request.name)
res = eval(request.mapfield["data"])
print res
for re in res:
print re
print re["Name"]
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.mapfield)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
cameramap_pb2_grpc.add_CameramapServicer_to_server(CameramapServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
Repeated
Not working on map, but add() is must to understand
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
#dictt = {'Name': 'Zara', 'Age': '7', 'Class': 'First'}
s = cameramap_pb2.Datatest(mapfield={'Name': 'Zara', 'Age': '7', 'Class': 'First'})
b = cameramap_pb2.HelloRequest()
bb = b.datatest.add()
bb.mapfield={'Name': 'Zara', 'Age': '7', 'Class': 'First'}
Real Cases
We would like to request a message as followed,
{
"api": "douban_movie",
"domain": "movie",
"res": [
{
"name": "卧虎藏龙",
"rating": 7.8,
"year": 2000
}
],
"time": "163ms"
}
Client Code
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
dictt = {'api': 'douban_movie', 'domain': 'movie', 'time': '163ms'}
listt = [{"name":"dragon", "rating":7.8, "year":"2000"}]
dictt["res"] = str(listt)
response = stub.SayHello(cameramap_pb2.HelloRequest(mapfield=dictt))
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
Remember, GRPC is strictly type, so you have to trasfer the generic type to string.
Server Code
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.mapfield
listt = eval(request.mapfield['res'])
# now it become a list
print listt[0]['rating'], type(listt[0]['rating'])
for v in listt:
print v
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.mapfield)
We use eval function to seperate the list, and accessing the rating to understand its type, float.
Since python supports generic types.
Seperate Json By String
Proto Code
message HelloRequest {
//string name = 1;
map<string, string> mapfield = 1;
repeated string res = 2;
}
Client Code
stub = cameramap_pb2_grpc.CameramapStub(channel)
dictt = {'api': 'douban_movie', 'domain': 'movie', 'time': '163ms'}
listt = {"name":"dragon", "rating": 7.8, "year":"2000"}
listts = []
listts.append(json.dumps(listt))
listt1 = {"name":"dragon1", "rating": 2.8, "year":"2001"}
listts.append(json.dumps(listt1))
response = stub.SayHello(cameramap_pb2.HelloRequest(mapfield=dictt, res=listts))
Servr Code
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.mapfield
mapdata = request.mapfield
print mapdata['api']
listts = request.res
# now it become a list
print listts
for listt in listts:
print listt, type(listt)
d = json.loads(listt)
print d['rating']
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.mapfield)
Seperate Json By Multiple Definition
In Proto
syntax = "proto3";
option java_multiple_files = true;
option java_package = "io.grpc.examples.helloworld";
option java_outer_classname = "HelloWorldProto";
package cameramap;
// The greeting service definition.
service Cameramap {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
rpc SayGoodBye (HelloRequest) returns (HelloReply) {}
}
message ResData {
string name = 1;
float rating = 2;
string year = 3;
}
// The request message containing the user's name.
message HelloRequest {
//string name = 1;
map<string, string> mapfield = 1;
repeated ResData res = 2;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}
Client Code
from __future__ import print_function
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
import json
def run():
channel = grpc.insecure_channel('localhost:50051')
stub = cameramap_pb2_grpc.CameramapStub(channel)
dictt = {'api': 'douban_movie', 'domain': 'movie', 'time': '163ms'}
s1 = cameramap_pb2.ResData(name ="haha", rating = 9.9, year = "2017")
s2 = cameramap_pb2.ResData(name ="haha1", rating = 9.99, year = "2017")
# it is allowed for the following way
tmp={}
tmp["name"]="haha2"
tmp["rating"] = 9.342
tmp["year"]="2017"
s = []
s.append(s1)
s.append(s2)
s.append(tmp)
hq = cameramap_pb2.HelloRequest(mapfield=dictt, res=s)
response = stub.SayHello(hq)
print("Greeter client received: " + response.message)
if __name__ == '__main__':
run()
Server Code
from concurrent import futures
import time
import math
import grpc
import cameramap_pb2
import cameramap_pb2_grpc
import json
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
class CameramapServicer(cameramap_pb2_grpc.CameramapServicer):
def SayHello(self, request, context):
print request.mapfield
mapdata = request.mapfield
print mapdata['api']
listts = request.res
# now it become a list
print listts
print '***************************'
for listt in listts:
print listt
print '-------------------'
return cameramap_pb2.HelloReply(message='Hello, %s!' % request.mapfield)
def serve():
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
cameramap_pb2_grpc.add_CameramapServicer_to_server(CameramapServicer(), server)
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
serve()
Retries Setting
proxyOpts := []grpc.DialOption{}
// since it's exponetially backoff, max delay time
var DefaultBackoffConfig = grpc.BackoffConfig{
MaxDelay: 12 * time.Second,
}
proxyOpts = append(proxyOpts, grpc.WithBackoffConfig(DefaultBackoffConfig))
// reconnect time out
proxyOpts = append(proxyOpts, grpc.WithTimeout(6000*time.Second))
// it must have, to block the initiation before established
proxyOpts = append(proxyOpts, grpc.WithBlock())
proxyOpts = append(proxyOpts, grpc.WithInsecure())
//conn, err := grpc.Dial(address, grpc.WithInsecure())
// conn, err := grpc.Dial(address, proxyOpts...)
conn, err := grpc.Dial(address, proxyOpts...)
if err != nil {
log.Fatalf("did not connect: %v", err)
}
Consul DNS Testing
ConsulDNS server in 192.168.51.129.
consul agent -dev -ui -server -node=consul-dev -client=192.168.51.129 -dns-port=53 -config-dir=/root/dnsconfig -domain=localdomain
In Other server, we need to setup nameserver.
root@ubuntu:~# cat /etc/resolv.conf
# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
nameserver 192.168.51.129
search localdomain
Adding Endpoints to Consul DNS Server.
root@ubuntu:~# cat dnsconfig/service.json
{
"services": [{
"id":"dotnetcoresample",
"name":"dotnetcoresample",
"tags":["dotnetcoresample"],
"address": "192.168.51.129",
"port": 50051
}]
}
However, port will not include the dns routing only IP takes effect.
In Any Server, try it.
root@ubuntu:~# ping dotnetcoresample.service.dc1.consul
PING dotnetcoresample.service.dc1.consul.localdomain (192.168.51.129) 56(84) bytes of data.
64 bytes from 192.168.51.129: icmp_seq=1 ttl=64 time=0.011 ms
64 bytes from 192.168.51.129: icmp_seq=2 ttl=64 time=0.039 ms
64 bytes from 192.168.51.129: icmp_seq=3 ttl=64 time=0.033 ms
or
root@ubuntu:~# ping dotnetcoresample.service
PING dotnetcoresample.service.localdomain (192.168.51.129) 56(84) bytes of data.
64 bytes from 192.168.51.129: icmp_seq=1 ttl=64 time=0.012 ms
64 bytes from 192.168.51.129: icmp_seq=2 ttl=64 time=0.040 ms
GRPC-LB for Aware Client
It only supports for grpc-go client, that will connect and record to etcd and consul as a metadata stored.
https://github.com/liyue201/grpc-lb
So for general purpose, such as consul-dns, might be a good solution.
