docker使用registry搭建本地镜像仓库实例详解

[root@k8smaster ~]# docker pull hub.c.163.com/library/registry:latest latest: Pulling from library/registry 25728a036091: Pull complete 0da5d1919042: Pull complete e27a85fd6357: Pull complete d9253dc430fe: Pull complete 916886b856db: Pull complete Digest: sha256:fce8e7e1569d2f9193f75e9b42efb07a7557fc1e9d2c7154b23da591e324f3d1 Status: Downloaded newer image for hub.c.163.com/library/registry:latest hub.c.163.com/library/registry:latest #registry镜像已经拉取下来 [root@k8smaster ~]# docker images REPOSITORY                                                        TAG        IMAGE ID       CREATED          SIZE hub.c.163.com/library/registry                                    latest     751f286bc25e   4 years ago      33.2MB 

[root@k8smaster ~]# docker history hub.c.163.com/library/registry:latest IMAGE          CREATED       CREATED BY                                      SIZE      COMMENT 751f286bc25e   4 years ago   /bin/sh -c #(nop)  CMD ["/etc/docker/registr…   0B       4 years ago   /bin/sh -c #(nop)  ENTRYPOINT ["/entrypoint.…   0B       4 years ago   /bin/sh -c #(nop) COPY file:7b57f7ab1a8cf85c…   155B       4 years ago   /bin/sh -c #(nop)  EXPOSE 5000/tcp              0B       4 years ago   /bin/sh -c #(nop)  VOLUME [/var/lib/registry]   0B       4 years ago   /bin/sh -c #(nop) COPY file:6c4758d509045dc4…   295B       4 years ago   /bin/sh -c #(nop) COPY file:b99d4fe47ad1addf…   22.8MB       4 years ago   /bin/sh -c set -ex     && apk add --no-cache…   5.61MB       4 years ago   /bin/sh -c #(nop)  CMD ["/bin/sh"]              0B       4 years ago   /bin/sh -c #(nop) ADD file:89e72bfc19e81624b…   4.81MB 

[root@k8smaster ~]# docker run -dit --restart=always --name=docker-registry -p 5000:5000 -v /docker/var/lib/registry:/var/lib/registry hub.c.163.com/library/registry:latest 3f8378272356bece1d690f16ad925bbd25a9ec12840d0612df2c45f84c27b3f5 [root@k8smaster ~]# docker ps CONTAINER ID   IMAGE                                                 COMMAND                  CREATED         STATUS         PORTS                                       NAMES 3f8378272356   hub.c.163.com/library/registry:latest                 "/entrypoint.sh /etc…"   4 seconds ago   Up 2 seconds   0.0.0.0:5000->5000/tcp, :::5000->5000/tcp   docker-registry 

[root@k8smaster ~]# ls /docker/var/lib/registry 

[root@k8sworker1 ~]# docker images REPOSITORY                                                TAG       IMAGE ID       CREATED         SIZE hub.c.163.com/library/wordpress                           latest    dccaeccfba36   4 years ago     406MB hub.c.163.com/library/tomcat                              latest    72d2be374029   4 years ago     292MB hub.c.163.com/library/mysql                               latest    9e64176cd8a2   4 years ago     407MB 

[root@k8sworker1 ~]# docker tag hub.c.163.com/library/wordpress 192.168.110.137:5000/boke/wordpress:latest [root@k8sworker1 ~]# docker tag hub.c.163.com/library/tomcat 192.168.110.137:5000/web/tomcat:v1 [root@k8sworker1 ~]# docker tag hub.c.163.com/library/mysql 192.168.110.137:5000/db/mysql:5.7 

[root@k8sworker1 ~]# docker push 192.168.110.137:5000/boke/wordpress:latest The push refers to repository [192.168.110.137:5000/boke/wordpress] Get https://192.168.110.137:5000/v2/: http: server gave HTTP response to HTTPS client 

[root@k8sworker1 ~]# systemctl status docker ● docker.service - Docker Application Container Engine Loaded: loaded (/usr/lib/systemd/system/docker.service; enabled; vendor preset: disabled) Active: active (running) since 二 2022-01-04 10:53:14 CST; 7h ago Docs: https://docs.docker.com Main PID: 1039 (dockerd) Memory: 1.2G CGroup: /system.slice/docker.service └─1039 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock #/usr/bin/dockerd有一个参数--insecure-registry用于指定不安全的仓库 [root@k8sworker1 ~]# /usr/bin/dockerd --help | grep insecure --insecure-registry list                  Enable insecure registry communication [root@k8sworker1 ~]# vim /usr/lib/systemd/system/docker.service #修改内容如下：添加了私有仓库地址和端口  --insecure-registry 192.168.110.137:5000 [root@k8sworker1 ~]# cat /usr/lib/systemd/system/docker.service | grep insecure-registry ExecStart=/usr/bin/dockerd --insecure-registry 192.168.110.137:5000 -H fd:// --containerd=/run/containerd/containerd.sock #重新加载配置文件并重启docker [root@k8sworker1 ~]# systemctl daemon-reload ; systemctl restart docker [root@k8sworker1 ~]# systemctl status docker ● docker.service - Docker Application Container Engine Loaded: loaded (/usr/lib/systemd/system/docker.service; enabled; vendor preset: disabled) Active: active (running) since 二 2022-01-04 17:57:26 CST; 14s ago Docs: https://docs.docker.com Main PID: 62817 (dockerd) Memory: 49.0M CGroup: /system.slice/docker.service └─62817 /usr/bin/dockerd --insecure-registry 192.168.110.137:5000 -H fd:// --containerd=/run/containerd/containerd.sock #此时，推送wordpress镜像到私有仓库成功 [root@k8sworker1 ~]# docker push 192.168.110.137:5000/boke/wordpress:latest The push refers to repository [192.168.110.137:5000/boke/wordpress] 53e16fa1f104: Pushed 562dd11ed871: Pushed ...... ddd6dcab19ff: Pushed 2c40c66f7667: Pushed latest: digest: sha256:ca4cf4692b7bebd81f229942c996b1c4e6907d6733e977e93d671a54b8053a22 size: 4078 

#新增"insecure-registries":["192.168.110.137:5000"] [root@k8sworker1 ~]# cat /etc/docker/daemon.json { "registry-mirrors": ["https://frz7i079.mirror.aliyuncs.com"], "insecure-registries":["192.168.110.137:5000"] } [root@k8sworker1 ~]# systemctl restart docker [root@k8sworker1 ~]# cat /etc/docker/daemon.json { "registry-mirrors": ["https://frz7i079.mirror.aliyuncs.com"], "insecure-registries":["192.168.110.137:5000"] } #推送tomcat镜像到私有仓库成功 [root@k8sworker1 ~]# docker push 192.168.110.137:5000/web/tomcat:v1 The push refers to repository [192.168.110.137:5000/web/tomcat] f79699072473: Pushed ...... fe40be59465f: Pushed cf4ecb492384: Pushed v1: digest: sha256:6241d7435b5c4e9d54be7d61e834836a71b1934b5403e01eff8768f0e2bcf210 size: 3045 [root@k8sworker1 ~]# docker push 192.168.110.137:5000/db/mysql:5.7 The push refers to repository [192.168.110.137:5000/db/mysql] 8129a85b4056: Pushed ...... 295d6a056bfd: Pushed 5.7: digest: sha256:c0806ac73235043de2a6cb4738bb2f6a74f71d9c7aa0f19c8e7530fd6c299e75 size: 2617 

[root@k8sworker1 ~]# curl 192.168.110.137:5000/v2/_catalog {"repositories":["boke/wordpress","db/mysql","web/tomcat"]} 

[root@k8sworker1 ~]# curl 192.168.110.137:5000/v2/boke/wordpress/tags/list {"name":"boke/wordpress","tags":["latest"]} 

[root@k8sworker1 ~]# yum -y install jq 已加载插件：fastestmirror base                                                                                                         ...... 已安装: jq.x86_64 0:1.6-2.el7 作为依赖被安装: oniguruma.x86_64 0:6.8.2-1.el7 完毕！ 

[root@k8sworker1 ~]# cat list_images_from_registries.sh #!/bin/bash file=$(mktemp) curl -s $1:5000/v2/_catalog | jq | egrep -v '\{|\}|\[|]' | awk -F\" '{print $2}' > $file while read aa ; do tag=($(curl -s $1:5000/v2/$aa/tags/list | jq | egrep -v '\{|\}|\[|]|name' | awk -F\" '{print $2}')) for i in ${tag[*]} ; do echo $1:5000/${aa}:$i done done < $file rm -rf $file [root@k8sworker1 ~]# bash list_images_from_registries.sh 192.168.110.137 192.168.110.137:5000/boke/wordpress:latest 192.168.110.137:5000/db/mysql:5.7 192.168.110.137:5000/web/tomcat:v1 

[root@k8sworker1 ~]# docker images REPOSITORY                                                TAG       IMAGE ID       CREATED         SIZE 

[root@k8sworker1 ~]# docker pull 192.168.110.137:5000/boke/wordpress:latest latest: Pulling from boke/wordpress 21f90b3df721: Pull complete 13ce341a48bc: Pull complete ...... 500c148f4d2b: Pull complete Digest: sha256:ca4cf4692b7bebd81f229942c996b1c4e6907d6733e977e93d671a54b8053a22 Status: Downloaded newer image for 192.168.110.137:5000/boke/wordpress:latest 192.168.110.137:5000/boke/wordpress:latest [root@k8sworker1 ~]# docker pull 192.168.110.137:5000/web/tomcat:v1 v1: Pulling from web/tomcat a2149b3f2ac2: Pull complete ...... 8dbb09972def: Pull complete Digest: sha256:6241d7435b5c4e9d54be7d61e834836a71b1934b5403e01eff8768f0e2bcf210 Status: Downloaded newer image for 192.168.110.137:5000/web/tomcat:v1 192.168.110.137:5000/web/tomcat:v1 [root@k8sworker1 ~]# docker images REPOSITORY                                                TAG       IMAGE ID       CREATED         SIZE 192.168.110.137:5000/boke/wordpress                       latest    dccaeccfba36   4 years ago     406MB 192.168.110.137:5000/web/tomcat                           v1        72d2be374029   4 years ago     292MB 

[root@k8smaster ~]# ls /docker/var/lib/registry docker [root@k8smaster ~]# ls /docker/var/lib/registry/docker/ registry [root@k8smaster ~]# ls /docker/var/lib/registry/docker/registry/ v2 [root@k8smaster ~]# ls /docker/var/lib/registry/docker/registry/v2/ blobs  repositories [root@k8smaster ~]# ls /docker/var/lib/registry/docker/registry/v2/repositories/ boke  db  web [root@k8smaster ~]# ls /docker/var/lib/registry/docker/registry/v2/repositories/boke/ wordpress [root@k8smaster ~]# ls /docker/var/lib/registry/docker/registry/v2/repositories/boke/wordpress/ _layers  _manifests  _uploads 

#给脚本授予可执行权限 [root@k8smaster ~]# chmod +x delete_docker_registry_image #设置私有仓库的镜像文件夹 [root@k8smaster ~]# export REGISTRY_DATA_DIR=/docker/var/lib/registry/docker/registry/v2/ #删除私有仓库的wordpress镜像 [root@k8smaster ~]# ./delete_docker_registry_image -i boke/wordpress:latest INFO     [2022-01-04 19:56:08,375]  Deleting /docker/var/lib/registry/docker/registry/v2/blobs/sha256/46/4676067592f277e1723e4ab4c588603df0b3dea762e22c354f7ada29b391cf10 ....... INFO     [2022-01-04 19:56:08,412]  Deleting /docker/var/lib/registry/docker/registry/v2/repositories/boke/wordpress #在docker客户端查看发现私有仓库里的wordpress镜像已经被删除了 [root@k8sworker1 ~]# bash list_images_from_registries.sh 192.168.110.137 192.168.110.137:5000/db/mysql:5.7 192.168.110.137:5000/web/tomcat:v1 

[root@k8smaster ~]# cat delete_docker_registry_image #!/usr/bin/env python """ Usage: Shut down your registry service to avoid race conditions and possible data loss and then run the command with an image repo like this: delete_docker_registry_image.py --image awesomeimage --dry-run """ import argparse import json import logging import os import sys import shutil import glob logger = logging.getLogger(__name__) def del_empty_dirs(s_dir, top_level): """recursively delete empty directories""" b_empty = True for s_target in os.listdir(s_dir): s_path = os.path.join(s_dir, s_target) if os.path.isdir(s_path): if not del_empty_dirs(s_path, False): b_empty = False else: b_empty = False if b_empty: logger.debug("Deleting empty directory '%s'", s_dir) if not top_level: os.rmdir(s_dir) return b_empty def get_layers_from_blob(path): """parse json blob and get set of layer digests""" try: with open(path, "r") as blob: data_raw = blob.read() data = json.loads(data_raw) if data["schemaVersion"] == 1: result = set([entry["blobSum"].split(":")[1] for entry in data["fsLayers"]]) else: result = set([entry["digest"].split(":")[1] for entry in data["layers"]]) if "config" in data: result.add(data["config"]["digest"].split(":")[1]) return result except Exception as error: logger.critical("Failed to read layers from blob:%s", error) return set() def get_digest_from_blob(path): """parse file and get digest""" try: with open(path, "r") as blob: return blob.read().split(":")[1] except Exception as error: logger.critical("Failed to read digest from blob:%s", error) return "" def get_links(path, _filter=None): """recursively walk `path` and parse every link inside""" result = [] for root, _, files in os.walk(path): for each in files: if each == "link": filepath = os.path.join(root, each) if not _filter or _filter in filepath: result.append(get_digest_from_blob(filepath)) return result class RegistryCleanerError(Exception): pass class RegistryCleaner(object): """Clean registry""" def __init__(self, registry_data_dir, dry_run=False): self.registry_data_dir = registry_data_dir if not os.path.isdir(self.registry_data_dir): raise RegistryCleanerError("No repositories directory found inside " \ "REGISTRY_DATA_DIR '{0}'.". format(self.registry_data_dir)) self.dry_run = dry_run def _delete_layer(self, repo, digest): """remove blob directory from filesystem""" path = os.path.join(self.registry_data_dir, "repositories", repo, "_layers/sha256", digest) self._delete_dir(path) def _delete_blob(self, digest): """remove blob directory from filesystem""" path = os.path.join(self.registry_data_dir, "blobs/sha256", digest[0:2], digest) self._delete_dir(path) def _blob_path_for_revision(self, digest): """where we can find the blob that contains the json describing this digest""" return os.path.join(self.registry_data_dir, "blobs/sha256", digest[0:2], digest, "data") def _blob_path_for_revision_is_missing(self, digest): """for each revision, there should be a blob describing it""" return not os.path.isfile(self._blob_path_for_revision(digest)) def _get_layers_from_blob(self, digest): """get layers from blob by digest""" return get_layers_from_blob(self._blob_path_for_revision(digest)) def _delete_dir(self, path): """remove directory from filesystem""" if self.dry_run: logger.info("DRY_RUN: would have deleted %s", path) else: logger.info("Deleting %s", path) try: shutil.rmtree(path) except Exception as error: logger.critical("Failed to delete directory:%s", error) def _delete_from_tag_index_for_revision(self, repo, digest): """delete revision from tag indexes""" paths = glob.glob( os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/tags/*/index/sha256", digest) ) for path in paths: self._delete_dir(path) def _delete_revisions(self, repo, revisions, blobs_to_keep=None): """delete revisions from list of directories""" if blobs_to_keep is None: blobs_to_keep = [] for revision_dir in revisions: digests = get_links(revision_dir) for digest in digests: self._delete_from_tag_index_for_revision(repo, digest) if digest not in blobs_to_keep: self._delete_blob(digest) self._delete_dir(revision_dir) def _get_tags(self, repo): """get all tags for given repository""" path = os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/tags") if not os.path.isdir(path): logger.critical("No repository '%s' found in repositories directory %s", repo, self.registry_data_dir) return None result = [] for each in os.listdir(path): filepath = os.path.join(path, each) if os.path.isdir(filepath): result.append(each) return result def _get_repositories(self): """get all repository repos""" result = [] root = os.path.join(self.registry_data_dir, "repositories") for each in os.listdir(root): filepath = os.path.join(root, each) if os.path.isdir(filepath): inside = os.listdir(filepath) if "_layers" in inside: result.append(each) else: for inner in inside: result.append(os.path.join(each, inner)) return result def _get_all_links(self, except_repo=""): """get links for every repository""" result = [] repositories = self._get_repositories() for repo in [r for r in repositories if r != except_repo]: path = os.path.join(self.registry_data_dir, "repositories", repo) for link in get_links(path): result.append(link) return result def prune(self): """delete all empty directories in registry_data_dir""" del_empty_dirs(self.registry_data_dir, True) def _layer_in_same_repo(self, repo, tag, layer): """check if layer is found in other tags of same repository""" for other_tag in [t for t in self._get_tags(repo) if t != tag]: path = os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/tags", other_tag, "current/link") manifest = get_digest_from_blob(path) try: layers = self._get_layers_from_blob(manifest) if layer in layers: return True except IOError: if self._blob_path_for_revision_is_missing(manifest): logger.warn("Blob for digest %s does not exist. Deleting tag manifest: %s", manifest, other_tag) tag_dir = os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/tags", other_tag) self._delete_dir(tag_dir) else: raise return False def _manifest_in_same_repo(self, repo, tag, manifest): """check if manifest is found in other tags of same repository""" for other_tag in [t for t in self._get_tags(repo) if t != tag]: path = os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/tags", other_tag, "current/link") other_manifest = get_digest_from_blob(path) if other_manifest == manifest: return True return False def delete_entire_repository(self, repo): """delete all blobs for given repository repo""" logger.debug("Deleting entire repository '%s'", repo) repo_dir = os.path.join(self.registry_data_dir, "repositories", repo) if not os.path.isdir(repo_dir): raise RegistryCleanerError("No repository '{0}' found in repositories " "directory {1}/repositories". format(repo, self.registry_data_dir)) links = set(get_links(repo_dir)) all_links_but_current = set(self._get_all_links(except_repo=repo)) for layer in links: if layer in all_links_but_current: logger.debug("Blob found in another repository. Not deleting: %s", layer) else: self._delete_blob(layer) self._delete_dir(repo_dir) def delete_repository_tag(self, repo, tag): """delete all blobs only for given tag of repository""" logger.debug("Deleting repository '%s' with tag '%s'", repo, tag) tag_dir = os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/tags", tag) if not os.path.isdir(tag_dir): raise RegistryCleanerError("No repository '{0}' tag '{1}' found in repositories " "directory {2}/repositories". format(repo, tag, self.registry_data_dir)) manifests_for_tag = set(get_links(tag_dir)) revisions_to_delete = [] blobs_to_keep = [] layers = [] all_links_not_in_current_repo = set(self._get_all_links(except_repo=repo)) for manifest in manifests_for_tag: logger.debug("Looking up filesystem layers for manifest digest %s", manifest) if self._manifest_in_same_repo(repo, tag, manifest): logger.debug("Not deleting since we found another tag using manifest: %s", manifest) continue else: revisions_to_delete.append( os.path.join(self.registry_data_dir, "repositories", repo, "_manifests/revisions/sha256", manifest) ) if manifest in all_links_not_in_current_repo: logger.debug("Not deleting the blob data since we found another repo using manifest: %s", manifest) blobs_to_keep.append(manifest) layers.extend(self._get_layers_from_blob(manifest)) layers_uniq = set(layers) for layer in layers_uniq: if self._layer_in_same_repo(repo, tag, layer): logger.debug("Not deleting since we found another tag using digest: %s", layer) continue self._delete_layer(repo, layer) if layer in all_links_not_in_current_repo: logger.debug("Blob found in another repository. Not deleting: %s", layer) else: self._delete_blob(layer) self._delete_revisions(repo, revisions_to_delete, blobs_to_keep) self._delete_dir(tag_dir) def delete_untagged(self, repo): """delete all untagged data from repo""" logger.debug("Deleting utagged data from repository '%s'", repo) repositories_dir = os.path.join(self.registry_data_dir, "repositories") repo_dir = os.path.join(repositories_dir, repo) if not os.path.isdir(repo_dir): raise RegistryCleanerError("No repository '{0}' found in repositories " "directory {1}/repositories". format(repo, self.registry_data_dir)) tagged_links = set(get_links(repositories_dir, _filter="current")) layers_to_protect = [] for link in tagged_links: layers_to_protect.extend(self._get_layers_from_blob(link)) unique_layers_to_protect = set(layers_to_protect) for layer in unique_layers_to_protect: logger.debug("layer_to_protect: %s", layer) tagged_revisions = set(get_links(repo_dir, _filter="current")) revisions_to_delete = [] layers_to_delete = [] dir_for_revisions = os.path.join(repo_dir, "_manifests/revisions/sha256") for rev in os.listdir(dir_for_revisions): if rev not in tagged_revisions: revisions_to_delete.append(os.path.join(dir_for_revisions, rev)) for layer in self._get_layers_from_blob(rev): if layer not in unique_layers_to_protect: layers_to_delete.append(layer) unique_layers_to_delete = set(layers_to_delete) self._delete_revisions(repo, revisions_to_delete) for layer in unique_layers_to_delete: self._delete_blob(layer) self._delete_layer(repo, layer) def get_tag_count(self, repo): logger.debug("Get tag count of repository '%s'", repo) repo_dir = os.path.join(self.registry_data_dir, "repositories", repo) tags_dir = os.path.join(repo_dir, "_manifests/tags") if os.path.isdir(tags_dir): tags = os.listdir(tags_dir) return len(tags) else: logger.info("Tags directory does not exist: '%s'", tags_dir) return -1 def main(): """cli entrypoint""" parser = argparse.ArgumentParser(description="Cleanup docker registry") parser.add_argument("-i", "--image", dest="image", required=True, help="Docker image to cleanup") parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", help="verbose") parser.add_argument("-n", "--dry-run", dest="dry_run", action="store_true", help="Dry run") parser.add_argument("-f", "--force", dest="force", action="store_true", help="Force delete (deprecated)") parser.add_argument("-p", "--prune", dest="prune", action="store_true", help="Prune") parser.add_argument("-u", "--untagged", dest="untagged", action="store_true", help="Delete all untagged blobs for image") args = parser.parse_args() handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(u'%(levelname)-8s [%(asctime)s]  %(message)s')) logger.addHandler(handler) if args.verbose: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) # make sure not to log before logging is setup. that'll hose your logging config. if args.force: logger.info( "You supplied the force switch, which is deprecated. It has no effect now, and the script defaults to doing what used to be only happen when force was true") splitted = args.image.split(":") if len(splitted) == 2: image = splitted[0] tag = splitted[1] else: image = args.image tag = None if 'REGISTRY_DATA_DIR' in os.environ: registry_data_dir = os.environ['REGISTRY_DATA_DIR'] else: registry_data_dir = "/opt/registry_data/docker/registry/v2" try: cleaner = RegistryCleaner(registry_data_dir, dry_run=args.dry_run) if args.untagged: cleaner.delete_untagged(image) else: if tag: tag_count = cleaner.get_tag_count(image) if tag_count == 1: cleaner.delete_entire_repository(image) else: cleaner.delete_repository_tag(image, tag) else: cleaner.delete_entire_repository(image) if args.prune: cleaner.prune() except RegistryCleanerError as error: logger.fatal(error) sys.exit(1) if __name__ == "__main__": main()