内网GitLab与PingCode打通

公司是内网Gitlab所以配置的webhook没办法直接跟pingcode交互。然后就衍生出了这个奇奇怪怪的姿势：拥有外网访问权限的机器上创建一个jenkins任务 每隔两分钟利用Gitlab API获取每个分支的commit以及合并请求构建成webhook内容访问PingCode提供的接口，再通过PingCode提供的RESTful API同步工作状态及登记工时

Python脚本主要干了以下事情：

1. 检测每个分支的提交，然后构建push的webhook提交到pingcode
   1. commit的title带上#+pingcode的任务编号(例:#D-123)关联提交次数
   2. merge_request带上#+pingcode的任务编号关联到Merge state
   3. commit的title内容带上$[0-99](例: $4)登记工时
   4. commit自动关联到任务的父任务
   5. 提交后自动将任务变更到“进行中”状态
2. 检测每个往develop提交的合并请求，然后构建merge_request的webhook提交到pingcode
   1. 状态为opened时，将任务变更到“已完成｜已修复”
   2. 状态为merged时，将任务变更到“待验收”
   3. 合并后@任务创建人提示已经合并

其他

PingCode的工时登记API只能由个人token访问，所以需要在PingCode后台创建一个个人鉴权的凭证，回调地址随便填一个(例：https://open.pingcode.com)

再将应用链接发送给其他用户让其授权，授权后得到一个Code，再用Code获取用户Token。

1. 获取Code
   1. 发送链接：https://open.pingcode.com/oauth2/authorize?response_type=code&client_id={应用的Client ID}
   2. 获取Code：得到其他用户把回调链接中的Code
2. 获取Token
   1. 访问API：https://open.pingcode.com/v1/auth/token?grant_type=authorization_code&client_id=tvNKZvWrdLDb&client_secret=xAyHtgVQwttiWYcYPMYzrVUa&code={其他用户的Code}
   2. 存储Token：将API返回的信息记录到本地JSON中

附脚本(pingcode.py)

#!/usr/bin/env python3
# encoding: utf-8
#

__Author__ = 'lipenghui'
__Date__ = '2021-12-30'

from datetime import datetime
from datetime import timedelta
from itertools import count
import os
import sys
import time

reload(sys)
sys.setdefaultencoding('utf8')
# need pip install python-gitlab
import gitlab
import json
import requests
import re

class PingCode():
    def __init__(self,url):
        self.STATE_TYPE_ING     =0 #正在
        self.STATE_TYPE_DONE    =1 #完成
        self.STATE_TYPE_CHECK   =2 #验收

        # 这部分状态在pingcode中不要删除
        self.STATE_OPEN="61737417429e0dc6408d3211"              # 打开
        self.STATE_NEW="61737417429e0dab368d320a"               # 新提交
        self.STATE_PENDING="6177cbd05108bc6c9ed202a3"           # 待办

        self.STATE_BUG_INPROGRESS="61737417429e0d5e638d320b"    # 处理中 普通提交改为此状态
        self.STATE_BUG_FIXED="61737417429e0d71ba8d320c"         # 已修复 合并状态为opened改为此状态

        self.STATE_TASK_INPROGRESS="61737417429e0d6b098d3212"   # 进行中 任务状态
        self.STATE_INSCEPTION="6182336daffb9b86101e85ff"        # 待验收 合并状态为merged改为此状态
        self.STATE_DONE="61737417429e0ddafb8d3213"              # 已完成 任务合并状态为opened改为此状态

        self.headers={'Content-Type':'application/Json'}
        self.url=url

        self.openApi="https://open.pingcode.com"
        self.client_id=""
        self.client_secret=""
        self.token="{鉴权Token}"
        #要写绝对路径，否则Jenkins执行时会找不到文件
        with open("pingcode.json",mode='r') as f:
            self.users=json.load(f)
    def getToken(self,userName=""):
        # TODO 保存token，时间戳到本地，过期的时候重新获取，目前token持续90天
        if False:
            body={
                "grant_type":"client_credentials",
                "client_id":"gFsPGXQoxElx",
                "client_secret":"oDowultNitJJejutdLrqhRAc"
            }
            www=requests.get(self.openApi+"/v1/auth/token",headers=self.headers,params=body)
            response=json.loads(www.text)
            self.token=response["access_token"]
        if self.users.has_key(userName):
            return self.users[userName]["access_token"]
        else:
            return self.token
    def push(self,content,commit,item):
        response=self.Send(json.dumps(content))
        response=json.loads(response.text)
        if response.has_key("code"):
            code=int(response["code"])
            if code==200 and item != None:
                 #登记工时
                workTimes=gitOperation.Get_workTimes(commit.title+commit.message)
                if len(workTimes)>0:
                    pingcode.registerManHours(commit.author_name,item["id"],workTimes[0].replace("$",""),commit.title)
    def merge(self,content,item,stateType,target):
        response=self.Send(json.dumps(content))
        response=json.loads(response.text)
        if target=="develop" and response.has_key("code"):
            code=int(response["code"])
            if code==200:
                pingcode.updateWorkItem(item["id"],item,stateType)
    def Send(self,gitData):
        response=requests.post(url=self.url,headers=self.headers,data=gitData)
        print(response.text)
        return response #json.loads(response)
    def getWorkItem(self,identifier):
        url=self.openApi+"/v1/agile/work_items"
        body={
            "identifier":identifier,
            "access_token":self.token
        }
        www=requests.get(url,headers=self.headers,params=body)
        response=json.loads(www.text)
        if response :
            if response.has_key("values"):
                return response["values"][0]
            else:
                print(response["message"])
        return None
    # 1.正在进行 2.已修复 3.待验收
    # 合并@    
    def updateWorkItem(self,id,item,stateType):
        itemState=item["state"]["id"]
        createById=item["created_by"]["id"]
        createByName=item["created_by"]["display_name"]
        # if (itemState==self.STATE_OPEN or itemState==self.STATE_NEW or itemState==self.STATE_PENDING) and stateType==self.STATE_TYPE_ING:
        if item["type"]=="bug":
            self.updateBug(id,item,stateType)
        elif item["type"]=="task":
            self.updateTask(id,item,stateType)

        # else:
            # print("workitem is in_progress or finish")
        content=""
        if stateType==self.STATE_TYPE_DONE:
            # content="[auto] [@e40b744b3b3449c29d8ac9c92347964a|巫佳翔] 已发起和并请求，请及时处理"
            pass
        elif stateType==self.STATE_TYPE_CHECK:
            content="[@"+createById+"|"+createByName+"] 已合并，请及时验收"
        if content!="":
            self.commentWorkItem(id,content)
    def updateBug(self,id,item,stateType):
        url=self.openApi+"/v1/agile/bugs/"+id
        state_id=self.STATE_BUG_INPROGRESS
        if  stateType==self.STATE_TYPE_ING:
            state_id=self.STATE_BUG_INPROGRESS
        elif stateType==self.STATE_TYPE_DONE:
            state_id=self.STATE_BUG_FIXED
        elif stateType==self.STATE_TYPE_CHECK:
            state_id=self.STATE_INSCEPTION
        param={
            "access_token":self.token
        }
        body={
            "state_id":state_id
        }
        www=requests.patch(url,headers=self.headers,params=param,data=json.dumps(body))
    def updateTask(self,id,item,stateType):
        url=self.openApi+"/v1/agile/tasks/"+id
        state_id=self.STATE_BUG_INPROGRESS
        if  stateType==self.STATE_TYPE_ING:
            state_id=self.STATE_TASK_INPROGRESS
        elif stateType==self.STATE_TYPE_DONE:
            state_id=self.STATE_DONE
        elif stateType==self.STATE_TYPE_CHECK:
            state_id=self.STATE_INSCEPTION
        param={
            "access_token":self.token
        }
        body={
            "state_id":state_id
        }
        www=requests.patch(url,headers=self.headers,params=param, data=json.dumps(body))
    def commentWorkItem(self,identerfi,content):
        url=self.openApi+"/v1/agile/work_items/"+identerfi+"/comments"
        body={
            "content":content,
            "access_token":self.token
        }
        www=requests.post(url, headers=self.headers,data=json.dumps(body))
        print("commentWorkItem:"+www.text)
    def registerManHours(self,userName,workItem,manHours,desp):
        """
        登记工时
        """
        userToken=self.getToken(userName)
        print(userName+" cost :"+" "+manHours +" hours on work item:"+workItem)
        url=self.openApi+"/v1/agile/workloads"
        manHours=float(manHours)
        param={
            "access_token":userToken
        }
        body={
            "work_item_id":workItem,
            "workload_type_id":"5cb7e7fffda1ce4ca0050002",
            "duration":manHours,
            "report_at":int(time.time()),
            "description":desp
        }
        www=requests.post(url,headers=self.headers,params=param, data=json.dumps(body))
class GitOperation():
    def __init__(self,project):
        self.project={
            "id": project.id,
            "name": project.name,
            "description": project.description,
            "web_url": project.web_url,
            "avatar_url": "",
            "git_ssh_url": project.web_url,
            "git_http_url": project.web_url,
            "namespace": project.namespace,
            "visibility_level": 0,
            "path_with_namespace": project.path_with_namespace,
            "default_branch": project.default_branch,
            "ci_config_path": "",
            "homepage": project.web_url,
            "url": project.web_url,
            "ssh_url": project.web_url,
            "http_url": project.web_url
            }
        self.repository= {
            "name": "rome",
            "url": project.web_url,
            "description": project.description,
            "homepage": project.web_url,
            "git_http_url": project.http_url_to_repo,
            "git_ssh_url": project.web_url,
            "visibility_level": 0
        }
    def Get_workTimes(self,content):
        """ 
        获取提交说明中的工时信息
        每次提交的工时记录需要低于10小时
        """
        pattern=re.compile(r'\$\d{,2}[\.\d{,1}]{,2}')
        workItems=pattern.findall(content)
        return workItems
    def Get_WorkItems(self,content):
        """ 获取提交说明中的任务编号"""
        pattern=re.compile(r'D-\d{,6}')
        workItems=pattern.findall(content)
        return workItems
    def Get_Git_Commit_Dict(self,commit):
        return {
            "id": commit.id,
            "parent":commit.parent_ids,
            "message": commit.message,
            "title": commit.title,
            "timestamp": commit.committed_date,
            "url": self.project["web_url"]+"/commit/"+commit.id,
            "author": {
                "name": commit.author_name,
                "email": commit.author_email
            },
            "added": [],
            "modified": [],
            "removed": []
        }
    def Get_Event_Merge(self,merge):
        """ 构建Merge_request的webhook格式请求json """
        request={
            "object_kind": "merge_request",
            "event_type": "merge_request",
            "user": {
                "id": merge.author["id"],
                "name": merge.author["name"],
                "username": merge.author["username"],
                "avatar_url":  merge.author["avatar_url"],
                "email": "[REDACTED]"
            },
            "project":self.project,
            "object_attributes": 
            {
                "assignee_id": merge.assignee,
                "author_id": merge.author["id"],
                "created_at": merge.created_at,
                "description": merge.description, 
                "head_pipeline_id": None,
                "id": merge.id,
                "iid": merge.iid,
                "last_edited_at": None,
                "last_edited_by_id": None,
                "merge_commit_sha": merge.merge_commit_sha,
                "merge_error": None,
                "merge_params": 
                {
                    "force_remove_source_branch": merge.force_remove_source_branch
                },
                "merge_status": merge.merge_status,
                "merge_user_id":None,
                "merge_when_pipeline_succeeds": merge.merge_when_pipeline_succeeds,
                "milestone_id": None,
                "source_branch": merge.source_branch,
                "source_project_id": merge.source_project_id,
                "state_id": 1,
                "target_branch": merge.target_branch,
                "target_project_id": merge.target_project_id,
                "time_estimate": 0,
                "title": merge.title,
                "updated_at": merge.updated_at,
                "updated_by_id": None,
                "url": merge.web_url,
                "source": self.project,
                "target": self.project,
                "work_in_progress": merge.work_in_progress,
                "total_time_spent": 0,
                "time_change": 0,
                "human_total_time_spent": None,
                "human_time_change": None,
                "human_time_estimate": None,
                "assignee_ids": [],
                "state": merge.state,
                "action": "open"
            },
            "labels": [
        
            ],
            "changes": {
                "merge_status": {
                    "previous": "unchecked",
                    "current": merge.merge_status
                }
            },
            "repository": self.repository
        }
        if merge.state=="opened":
            pass
        elif merge.state=="closed":
            pass
        elif merge.state=="merged":
            # print(str(merge.merged_by["id"])+"合的")
            request["object_attributes"]["merge_user_id"]= merge.merged_by["id"]
            # print(request)
        return request
    def Get_Event_Push(self,branchName,commit):
        """ 构建push的webhook格式请求json """
        branchName=branchName.replace("优化","youhua")
        gitCommit=self.Get_Git_Commit_Dict(commit)
        commits=[]
        commits.append(gitCommit)
        return  {
            "object_kind": "push",
            "event_name":"push",
            "before": commit.parent_ids[0],
            "after": commit.id,
            "ref": "refs/heads/"+branchName,
            "checkout_sha": commit.id,

            "user_id": 35,
            "user_name": commit.author_name,
            "user_username": commit.author_name,
            "user_email": commit.author_email,
            "user_avatar": "https://www.gravatar.com/avatar/e96f394d1e9a973845ee60a6a73535de?s=80&d=identicon",
            # "project_id": self.project_id,
            "project": self.project,
            "repository": self.repository,
            "commits": commits,
            "total_commits_count": 1,
            "push_options": {}
        }
class GitlabAPI():
    def __init__(self, *args, **kwargs):
        #绝对路径
        if os.path.exists("python-gitlab.cfg"):
            self.gl = gitlab.Gitlab.from_config('lipenghui', ['python-gitlab.cfg'])
        # elif os.path.exists(os.getenv('HOME')+'/.python-gitlab.cfg'):
        #     self.gl = gitlab.Gitlab.from_config('lipenghui', [os.getenv('HOME') + '/.python-gitlab.cfg'])
        else:
            print('You need to make sure there is a file named "/python-gitlab.cfg" or "~/.python-gitlab.cfg"')
            sys.exit(5)
if __name__ == '__main__':

    api="https://apps.pingcode.com/gitlab/api/webhooks/{PingCode侧GitLab的ClientId}"
    pingcode=PingCode(api)

    gitApi = GitlabAPI()
    project = gitApi.gl.projects.get(1)
    gitOperation=GitOperation(project)
    tmpNow=datetime.now()
    if True:
				#时区问题 -8
        now=tmpNow+timedelta(hours=-8,minutes=-2)
        last=now.strftime('%Y-%m-%dT%H:%M')#datetime.strptime(,'%YYYY-%MM-%DD %HH:%MM').isoformat()
        if True: # commit
            branches=project.branches.list(per_page=100)
            print("["+last+"]=> ready to sync commits into pingcode...")
            for branch in branches:
                # print("check commit:"+branch.name+" Time:"+last)
                commits=project.commits.list(ref_name=branch.name,since=last)
                for commit in commits:                            
                    identifiers=gitOperation.Get_WorkItems(commit.title)
                    item=None
                    for identifier in identifiers:
                        item=pingcode.getWorkItem(identifier)
                        #关联父任务
                        print("identifier:"+identifier)
                        if item and item.has_key("parent") and item["parent"]:
                            commit.message=commit.message+" #"+item["parent"]["identifier"]
                        pingcode.updateWorkItem(item["id"],item,pingcode.STATE_TYPE_ING)
                       
                    # 同步提交日志
                    content=gitOperation.Get_Event_Push(branch.name,commit)
                    pingcode.push(content,commit,item)
        if True : # merge
            # 不知道为什么gitlab服务器上会少6分钟
            now=tmpNow+timedelta(hours=-8,minutes=-8)
            last=now.strftime('%Y-%m-%dT%H:%M')#datetime.strptime(,'%YYYY-%MM-%DD %HH:%MM').isoformat()
            print("["+last+"]=> ready to sync merges into pingcode...")
            merges= gitApi.gl.mergerequests.list(scope="all",updated_after=last)
            for merge in merges:
                mrg=project.mergerequests.get(merge.iid)
                commits=mrg.commits()
                if len(commits)>1:
                    for commit in commits:
                        isPingcodeWorkItem=gitOperation.Get_WorkItems(commit.title)
                        if len(isPingcodeWorkItem)>0:
                            itemId=isPingcodeWorkItem[0]
                            if merge.title.find(itemId)<0:
                                merge.title+=" "+itemId
                                merge.description+=" "+itemId
                # print(mrg.title+" =>"+mrg.state+"=> commits:"+str(len(commits))+" => "+merge.description)
                stateType=pingcode.STATE_TYPE_ING
                if mrg.state=="merged":
                    stateType=pingcode.STATE_TYPE_CHECK
                elif mrg.state=="closed":
                    stateType=pingcode.STATE_TYPE_ING
                elif mrg.state=="opened":
                    stateType=pingcode.STATE_TYPE_DONE
                # "D" 是pingcode后台项目的标识符 固定格式:D-123456
                identifiers=gitOperation.Get_WorkItems(merge.title)
                for identifier in identifiers:
                    item=pingcode.getWorkItem(identifier)
                    Content=gitOperation.Get_Event_Merge(merge)
                    pingcode.merge(Content,item,stateType,merge.target_branch)

PingCode配置文件(pingcode.json)

{
	"{GitLab账号}":{
		"name":"李鹏辉",
		"access_token": "{接口返回}",
		"token_type": "{接口返回}",
		"expires_in": "{接口返回}",
		"refresh_token": "{接口返回}",
		"code":"{其他用户的Code}"
	}
}

Json配置文件两部分[Key:Value]：

• key:其他用户在Gitlab上的用户名
• value:
  • name:用户名字
  • PingCode的Token信息(获取方法见后述)
    • access_token
    • token_type
    • expires_in：过期时间，默认90天
    • refresh_token：刷新Token的参数

GitLab配置文件(gitlab.cfg)

[global]
default ={GitLab登录账号}
ssh_verify = False
timeout = 8
[{GitLab登录账号}]
url = http://{GitLab仓库地址}
private_token = {GitLab个人账号创建的Token}
api_version = 4