# coding: utf-8 from operator import itemgetter from urllib import parse import requests import datetime as dt # import maya import logging logging.basicConfig(filename='harbor_clean.txt', filemode="w", level=logging.INFO) logger = logging.getLogger(__name__) """ 清理 Harbor 仓库的老镜像 """ class HarborCleaner(object): delete_status = { 200: "Delete tag successfully.", 400: "Invalid repo_name.", 401: "Unauthorized.", 403: "Forbidden.", 404: "Repository or tag not found.", } def __init__(self, user: str, password: str, hostname: str, port: int, use_https=True): scheme = "https" if use_https else "http" Api_base = F " {} scheme: // hostname {}: {Port} / API " self.search_api = api_base + " ? / Search key_word Q = {} " self.projects_api = api_base + " / Projects " self.repository_query_api api_base + = " / Repositories? PROJECT_ID PROJECT_ID = {} " # repo_name generally "project_name / repo_name" format, escaped must be done (because the middle slashes) self.repository_tags_api = api_base + " / Repositories /} {repo_name / Tags " self.repository_tag_api = Self.repository_tags_api + "/} {Tag " self.session = requests.Session () self.session.verify = False # If the company is using a self-signed certificate, SSL can not be verified, it is necessary to set this self.session.headers = { " the Accept " : " file application / JSON " } self.session.auth = (User, password) DEF get_all_projects (Self): RESP = self.session.get (self.projects_api) Success = == 200 is resp.status_code return { " Success ": success, "data": resp.json() if success else resp.text } def get_all_repos(self, project: dict): url = self.repository_query_api.format(project_id=project['project_id']) resp = self.session.get(url) success = resp.status_code == 200 return { "success": success, "data": resp.json() if success else resp.text } def get_all_tags(self, repo: dict): """repo_name 需要做转义""" repo_name = parse.quote(repo['name'], safe="") url = self.repository_tags_api.format(repo_name=repo_name) resp = self.session.get(url) success = resp.status_code == 200 return { "success": success, "data": resp.json() if success elseresp.text } DEF get_tags_except_lastest_n (Self, the repo: dict, n: int): "" " Get all tags except the n tag of the latest " "" # If the mirror is smaller than the number of tags n + 1, indicating that the mirror is clean, you do not need to clean up. IF the repo [ ' tags_count ' ] <= n-+. 1: # + 1'd are repeated because latest Tag return [] Result = self.get_all_tags (the repo) Tags: List = Result [ ' Data ' ] for Tag in Tags: # tag [ 'time'] = maya.MayaDT.from_iso8601 (tag [ 'created' '2019-04-09T11: 33 is: 49.296960745Z' # # Python comes analytic function, can only handle six decimal places, cut off the excess of the following three timestamp Tag = [ ' Created ' ] [: -. 4] + ' the Z ' Tag [ ' Time ' ] = dt.datetime.strptime (timestamp, R & lt ' % Y-dT%%% M- H:% M:% S.% fZ ' ) tags.sort (Key = itemgetter ( ' Time ' )) # using the time ordered keys in situ return Tags [: -. 1-n-] # Expect the latest n-Tags, because -1 is repeated latest Tag DEF soft_delete_tag (Self, the repo: dict, Tag: dict): """repo_name need to do to escape this deletion, the need to conduct a GC, in order to truly clean up the available space. "" " Repo_name = parse.quote (the repo [ ' name ' ], Safe = " " ) URL = self.repository_tag_api.format (repo_name = repo_name, Tag Tag = [ ' name ' ]) RESP = self.session.delete ( URL) return { " Success " : == 200 is resp.status_code , " Message " : self.delete_status.get (resp.status_code) } DEFsoft_delete_all_tags_except_latest_n (Self, n): "" " , to remove all the tags from each warehouse, except only the latest all tags outside the n Tag " "" res_projects = self.get_all_projects () IF Not res_projects [ ' Success ' ] : logger.warning ( " faild All Projects to GET, Message: {} " .format (res_projects [ ' Data ' ])) logger.info ( " WE have have Projects {} " .format (len (res_projects [ ' Data ']))) for p in res_projects['data']: res_repos = self.get_all_repos(p) if not res_projects['success']: logger.warning("faild to get all repos in project: {}, message: {}".format(p['name'], res_repos['data'])) logger.info("we have {} repos in project:{}".format(len(res_repos['data']), p['name'])) for repo in res_repos['data']: logger.info("deal with repo: {}".format(repo['name'])) old_tags = self.get_tags_except_lastest_n(repo, n) logger.info("we have {} tags to delete in repo: {}".format(len(old_tags), repo['name'])) for tag in old_tags: logger.info("try to delete repo:{}, tag: {}, create_time: {}".format(repo['name'], tag['name'], tag['created'])) result = self.soft_delete_tag(repo, tag) if result['success']: logger.info("success delete it.") else: logger.warning("delete failed!, message: {}".format(result['Message ' ])) IF the __name__ == " __main__ " : # 1. a soft delete restful api harbor by the harbor_cleaner = HarborCleaner ( User = " ADMIN " , password = " Admin123 " , hostname = " reg.harbor.com " , Port = 8321 ) harbor_cleaner.soft_delete_all_tags_except_latest_n ( 10) # each mirror retaining only the latest ten Tag #2. Make a GC, removed all soft-deleted ImagesRF Royalty Free # previous 2.1 harbor 1.7 version, you need to stop in order to GC "" " cd / volume1 / Docker / Harbor / Harbor Docker Compose-STOP # Stop # following tag 'v2 .6.2-v1.4.0 'need to replace the registry-photon image of the currently used version # --dry-run representation attempted GC, log output is consistent with the official gc, can be used to identify problems early docker run -it --name --rm --volumes from Registry-gc VMware / Registry-Photon: Garbage v2.6.2-v1.4.0-the collect the --dry-RUN /etc/registry/config.yml # official gc, gc this will really fall It has been soft-deleted image Docker RUN -it --name-gc --rm --volumes from VMware Registry / Registry-Photon: Garbage v2.6.2-v1.4.0-the collect /etc/registry/config.yml "" " # 2.2 harbor 1.7+ online GC can restful api or automatically at regular intervals by GC.
After Harbor GC clean up and remove the mirror, did not release the disk space of 1 , because after you remove a mirror and GC Harbor cleanup, disk space is not released. Because the mirror we push a lot of the same labels, Docker image referenced by the label, only a summary by the logo. This means that if myImage use different markers to push the two images, display them within the DR, which will consist of two different digests identity. The last push Images is current. Docker a mirror layers, each of which layers is associated with a blob. The blob is the most occupied stored files; GC will clean up those files. Being a reference to the above description of each image are stored, because, we repeat 10 times submitted that a label 10 there will be referenced in the DR, the label can only get tag. While the other nine can only be acquired with a digest. Simply means that because of excessive repetition of the same image tag to submit the number of leads. 2 , the solution 1, edit the Common / config / Registry / config.yml file this file in the installation directory under the harbor, close purpose is to prohibit authentication 2, modify docker- compose.yml file file directory in harbor installation, the purpose of this document is to modify the exposure registry port port out, add a red box configuration, pay attention to the format. 3 , reconfigure the harbor, making the configuration take effect the following command Docker - Compose Down Docker -compose up -d 4 , to clean up the list of deleted unused execute the following command Docker RUN --network = " Host " -it -v / the Data / Registry: / Registry -e REGISTRY_URL = HTTP: //127.0.0.1: 5000 mortensrasmussen / Docker -manifest--registry cleanup 5 , clear up to now is no longer associated with the blob to delete the list of executing the following command Docker RUN -it --name gc --rm --volumes- from Registry VMware / Registry-Photon: v2.6.2- Garbage the collect-V1.4.0 / etc / Registry / config.yml . 6, the configuration steps 1 and 2 back to the initial state changes, and restart the harbor.