持续创作，加速成长！这是我参与「掘金日新计划 · 10 月更文挑战」的第25天，点击查看活动详情
前言
昨天的文章中，我们描述了python flask部署python后端程序的原理，简而言之python flask虽然完成了部分server的功能，但是由于性能以及安全性较差，因此不适用于生产环境，由于请求经常以http服务的形式体现，因此仅仅使用uwsgi + python flask的形式不足以满足日常需求，引入nginx使得后端访问方式更加灵活与安全，如使用代理等技术，能够有效隐藏真实服务端口等，更加利于拓展。综上，python flask结合uwsgi与nginx是一个成熟的python web服务框架。今天将对之前的程序进行重构，并完成线上部署。
项目结构
其中程序分为app.py，文件夹中，config文件配置了neo4j数据库的链接方式，Controller文件夹保存了主要服务的实现，data文件夹是原始数据与数据插入脚本，Dto定义了部分数据传输的格式等，Utils文件夹保存了一些数据库链接、数据处理等基本工具单元。
neo4j.ini
[Neo4J]
REMOTE_URL=http://127.0.0.1:7474
REMOTE_PASS=XXX
REMOTE_USER=neo4j
其中REMOTE_URL是neo4j的连接地址，REMOTE_PASS是数据库连接密码，REMOTE_USER为用户名。
searchController.py
from Utils.connectUtils import connect2Neo4J
from Dto.ResultDto import ResultDto
from Utils.enumCode import Code
from Utils.enumMsg import Msg
from Utils.dataFormatUtils import dataFormat
class search4Node():
def __init__(self):
self.conn = connect2Neo4J().conn
self.maxNodes = 300
def getAllNodes(self, labels = None, properties = None, limit = None):
cypher = "match (n) return n"
if labels:
cypher = f"match (n:{labels}) return id(n),n"
if properties:
cypher = "match (n:{0}{{name:'{1}'}}) return id(n),n".format(labels,properties)
if limit:
cypher += ' limit {}'.format(limit)
else:
cypher = f"match (n) return n limit {self.maxNodes}"
print(cypher)
result = self.conn.run(cypher)
return result.data()
def searchFirstLevel(self):
result = self.conn.run("match (n:company{name:'cisco'}) -[r]-> (m:product) return m,r,n limit 20")
return result.data()
def get2PointRel(self, nodeA, nodeB, propertiesA=None, propertiesB=None, limit=50):
try:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}) return id(m),m ,id(r),r,id(n),n"
if propertiesA is not None and propertiesB is None:
cypher = f"match (n:{nodeA}{{name:'{propertiesA}'}}) -[r]-> (m:{nodeB}) return id(m),m ,id(r),r,id(n),n"
if propertiesB and propertiesA is None:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) return id(m),m ,id(r),r,id(n),n"
if propertiesA is not None and propertiesB is not None:
cypher = f"match (n:{nodeA}{{name:'{propertiesA}'}}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) return id(m),m ,id(r),r,id(n),n"
if limit == None:
limit = 10
cypher += ' limit {}'.format(limit)
else:
cypher += ' limit {}'.format(limit)
print(cypher)
result = self.conn.run(cypher)
result = result.data()
result = dataFormat().formatPointRelations(result)
return ResultDto(code=Code.Success.value, msg=Msg.Success.value,data=result).retrunMsg
except Exception as e:
print(e)
return ResultDto(code=Code.ServiceError.value, msg="{}:{}".format(Msg.ServiceError.value, e),
data="节点信息查询失败").retrunMsg
def get3PointRel(self, nodeA, nodeB, nodeC, propertiesA=None, propertiesB=None, propertiesC=None, limit=10):
try:
cypher = ""
if propertiesA is not None and propertiesB is None and propertiesC is None:
cypher = f"match (n:{nodeA}{{name:'{propertiesA}'}}) -[r]-> (m:{nodeB}) - [t]->(o:{nodeC}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is None and propertiesB is not None and propertiesC is None:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) - [t]->(o:{nodeC}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is None and propertiesB is None and propertiesC is not None:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}) - [t]->(o:{nodeC}{{name:'{propertiesC}'}}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is not None and propertiesB is not None and propertiesC is None:
cypher = f"match (n:{nodeA}{{name:'{propertiesA}'}}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) - [t]->(o:{nodeC}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is not None and propertiesB is not None and propertiesC is None:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) - [t]->(o:{nodeC}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is None and propertiesB is not None and propertiesC is not None:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) - [t]->(o:{nodeC}{{name:'{propertiesC}'}}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is not None and propertiesB is not None and propertiesC is not None:
cypher = f"match (n:{nodeA}{{name:'{propertiesA}'}}) -[r]-> (m:{nodeB}{{name:'{propertiesB}'}}) - [t]->(o:{nodeC}{{name:'{propertiesC}'}}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if propertiesA is None and propertiesB is None and propertiesC is None:
cypher = f"match (n:{nodeA}) -[r]-> (m:{nodeB}) - [t]->(o:{nodeC}) return id(n) ,n ,id(m),m, id (o), o ,id(r) ,r ,id(t), t"
if limit == None:
limit = 10
cypher += ' limit {}'.format(limit)
else:
cypher += ' limit {}'.format(limit)
result = self.conn.run(cypher)
result = dataFormat().formatPointRelations(result.data())
return ResultDto(code=Code.Success.value, msg=Msg.Success.value,data=result).retrunMsg
except Exception as e:
return ResultDto(code=Code.ServiceError.value, msg="{}:{}".format(Msg.ServiceError.value, e),
data="节点信息查询失败").retrunMsg
数据查询部分和之前的类似，依旧使用的是cypher语句完成查询操作，根据参数不同生成不同的查询语句完成数据库查询操作。
ResultDto.py
class ResultDto(object):
'''
返回结果
'''
def __init__(self,code,msg,data):
self.code = code
self.msg = msg
self.data = data
self.ReturnMsg()
def ReturnMsg(self):
self.retrunMsg = {"code": self.code, "msg": self.msg, "data": self.data}
定义了数据的返回形式，保护状态码，消息以及返回数据。
connectUtils.py
from configparser import RawConfigParser
from py2neo import Graph, Node, Relationship, NodeMatcher, NodeMatch,RelationshipMatcher
class connect2Neo4J():
def __init__(self):
self.getNeoCfg()
self.conn = Graph(self.REMOTE_URL, auth=(self.REMOTE_USER, self.REMOTE_PASS))
def getNeoCfg(self):
config = RawConfigParser()
# 获取配置文件的真实路径
path = r"config/neo4j.ini"
config.read(path, encoding="utf-8")
self.REMOTE_URL = config.get("Neo4J","REMOTE_URL")
self.REMOTE_PASS = config.get("Neo4J","REMOTE_PASS")
self.REMOTE_USER = config.get("Neo4J","REMOTE_USER")
def creatNode(self, labels, properties):
insertNode = Node(labels, name=properties)
self.conn.create(insertNode)
return insertNode
def creatRelationship(self, nodeA, nodeB, properties):
relation = Relationship(nodeA, properties, nodeB)
self.conn.create(relation)
def searchNode(self, labels, limit=None):
if limit:
data = NodeMatch(self.conn, labels=frozenset({'{}'.format(labels)})).limit(limit)
else:
node_matcher = NodeMatcher(self.conn)
data = node_matcher.match(labels)
return data
def getRelations(self,nodes = None,r_type = None,limit = None):
relation = RelationshipMatcher(self.conn)
relationData = relation.match(nodes=nodes, r_type='{}'.format(r_type)).limit(limit)
return relationData
数据库连接工具类，首先加载配置文件，然后完成数据库的连接操作，生成数据库连接对象。
dataFormatUtils.py
数据处理的工具类
from Utils.connectUtils import connect2Neo4J
class dataFormat(object):
def __init__(self):
self.vlunCategory = {'company': 0, 'product': 1, 'cvenumberNode': 2}
self.categories = [ {"name": "companyElement", "keyword": {}, "base": "companyElement"}, {"name": "productElement", "keyword": {}, "base": "productElement"}, {"name": "cvenumberElement", "keyword": {}, "base": "cvenumberElement"}, ]
def formatEchartNodes(self, itemList):
resList = []
for index, item in enumerate(itemList):
category = str(item["n"]).split(" {name:")[0].split(":")[1]
categoryIndex = self.vlunCategory[category]
name = item["n"].get("name")
resList.append({
"name": f"{name}",
"value": 1,
"category": categoryIndex
})
return resList
def formatG6Nodes(self):
pass
def formatG6Relations(self, itemList):
nodeIdDic = {}
combos = []
nodes = []
edges = []
testLine = itemList[0]
if testLine.get("r") != None:
combos.append({"id": 'a', "label": 'company'})
combos.append({"id": 'b', "label": 'product'})
if testLine.get("t") != None:
combos.append({"id": 'c', "label": 'cveNumber'})
for line in itemList:
for k, v in line.items():
if k == "r":
company = str(v).split(")-[:拥有 {}]->(")[0].lstrip("(")
product = str(v).split(")-[:拥有 {}]->(")[1].rstrip(")")
if company in nodeIdDic:
company_id = nodeIdDic.get(company)
nodes.append({"id": company_id, "comboId": "a"})
else:
nodeIdDic[company] = len(nodeIdDic)
nodes.append({"id": len(nodeIdDic), "comboId": "a"})
if product in nodeIdDic:
product_id = nodeIdDic.get(product)
nodes.append({"id": product_id, "comboId": "b"})
else:
nodeIdDic[product] = len(nodeIdDic)
nodes.append({"id": len(nodeIdDic), "comboId": "b"})
edges.append({"source": nodeIdDic.get(company), "target": nodeIdDic.get(product)})
if k == "t":
product = str(v).split("-[:暴露漏洞 {}]->(")[0].lstrip("(")
cveNumber = str(v).split("-[:暴露漏洞 {}]->(")[1].rstrip(")")
if product in nodeIdDic:
product_id = nodeIdDic.get(product)
nodes.append({"id": product_id, "comboId": "b"})
else:
nodeIdDic[product] = len(nodeIdDic)
nodes.append({"id": len(nodeIdDic), "comboId": "b"})
if cveNumber in nodeIdDic:
cveNumber_id = nodeIdDic.get(cveNumber)
nodes.append({"id": cveNumber_id, "comboId": "b"})
else:
nodeIdDic[cveNumber] = len(nodeIdDic)
nodes.append({"id": len(nodeIdDic), "comboId": "c"})
edges.append({"source": nodeIdDic.get(product), "target": nodeIdDic.get(cveNumber)})
return {"nodes": nodes, "edges": edges, "combos": combos}
def formatPointRelations(self, result):
nodes = []
edges = []
tempDic = {}
for line in result:
if line.get("n"):
# 第一层
# Node('company', name='teclib-edition')
label = str(line.get("n")).split(" {name: '")[0].split(":")[1]
value = str(line.get("n")).split(" {name: '")[1].rstrip("'})")
if value not in tempDic:
# MATCH (n:company{name:"cisco"})-[r]->() RETURN COUNT(r)
childrenNum = \
connect2Neo4J().conn.run(f"MATCH (n:{label}{{name:'{value}'}})-[r]->() RETURN COUNT(r)").data()[ 0].get("COUNT(r)")
node = {
"id": value,
"name": value,
"level": 0,
"childrenNum": childrenNum,
"tag": value
}
nodes.append(node)
tempDic[value] = 1
if line.get("m"):
# 第二层
label = str(line.get("m")).split(" {name: '")[0].split(":")[1]
value = str(line.get("m")).split(" {name: '")[1].rstrip("'})")
if value not in tempDic:
childrenNum = \
connect2Neo4J().conn.run(f"MATCH (n:{label}{{name:'{value}'}})-[r]->() RETURN COUNT(r)").data()[
0].get("COUNT(r)")
tags = str(line.get("n")).split(" {name: '")[1].rstrip("'})")
edge = {
"source": tags,
"target": value
}
edges.append(edge)
node = {
"id": value,
"name": value,
"level": 1,
"childrenNum": childrenNum,
"tag": value,
"isLeaf": False,
"tags": [
tags
]
}
nodes.append(node)
tempDic[value] = 1
if line.get("o"):
# 第三层
label = str(line.get("o")).split(" {name: '")[0].split(":")[1]
value = str(line.get("o")).split(" {name: '")[1].rstrip("'})")
if value not in tempDic:
tags = str(line.get("m")).split(" {name: '")[1].rstrip("'})")
edge = {
"source": tags,
"target": value
}
edges.append(edge)
node = {
"id": value,
"name": value,
"level": 2,
"isLeaf": True,
"tags": [
tags
]
}
nodes.append(node)
tempDic[value] = 1
return {"nodes": nodes, "edges": edges}
根据前端小姐姐的新的数据返回格式要求，更新了返回数据处理类，需要啥格式数据，制造啥数据格式。
enumCode.py
from enum import Enum
class Code(Enum):
Error = 4001
NameOrPwdError = 4002
ParameterError = 4003
NameNotExist = 4004
ServiceError = 4005
OriPasswordError = 4006
UserNameExist = 4007
RandomCodeError = 4009
TokenError = 4010
TokenExpire = 1001
TokenParsingFailed = 1002
Success = 200
enumMsg.py
from enum import Enum
class Msg(Enum):
Error = "失败"
Success = "成功"
NameOrPwdError = "用户名或密码错误"
TokenExpire = "Token已过期"
TokenParsingFailed = "Token解析失败"
ParameterError = "参数错误"
NameNotExist = "用户名不存在"
ServiceError = "内部服务异常"
OriPasswordError = "原始密码错误"
UserNameExist = "用户名重复"
RandomCodeError = "验证码错误"
TokenError = "token过期或不正确"
定义了一些错误类型和返回码。
app.py
from flask import Flask, request
from Utils.enumCode import Code
from Controller.searchController import search4Node
from Utils.dataFormatUtils import dataFormat
from Dto.ResultDto import ResultDto
app = Flask(__name__)
@app.route('/getRelations', methods=['GET'])
def getRelations():
nodeA = request.args.get("nodeA")
nodeB = request.args.get("nodeB")
nodeC = request.args.get("nodeC")
nodeAprop = request.args.get("nodeAprop")
nodeBprop = request.args.get("nodeBprop")
nodeCprop = request.args.get("nodeCprop")
limit = request.args.get("limit")
if nodeC == None and nodeCprop == None:
return search4Node().get2PointRel(nodeA, nodeB,nodeAprop, nodeBprop,limit)
else:
return search4Node().get3PointRel(nodeA, nodeB, nodeC, nodeAprop, nodeBprop, nodeCprop, limit)
if __name__ == '__main__':
app.run(host='127.0.0.1',port=5010)
主程序，目前只写了一个接口，完成了节点和关系的查询操作
项目通过uwsgi和nginx部署之后，在远端即可用postman完成调用，如下：
但是存在一个问题，neo4j的查询，返回50个节点基本是按照自己的处理逻辑，如何返回想要的，需要在研究一下，明天将使用redis完成前端初始页面的展示，蟹蟹~
