Skip to content

Containerd cannot pull image from insecure registry #3847

New issue
New issue
Closed
containerd/cri
#1345
Closed
Containerd cannot pull image from insecure registry#3847
containerd/cri
#1345
Labels
kind/bug
@qianzhangxa

Description

@qianzhangxa
qianzhangxa
opened on Nov 25, 2019

Description
I deployed Harbor (172.17.1.201) in my Kubernetes cluster and pushed an image (172.17.1.201/library/alpine) into it. When I tried to manually pull the image from a worker node (it uses containerd as container runtime and there is no Docker on this node at all) of my Kubernetes cluster, it failed:

$ crictl pull 172.17.1.201/library/alpine 
FATA[0000] pulling image failed: rpc error: code = Unknown desc = failed to resolve image "172.17.1.201/library/alpine:latest": no available registry endpoint: failed to do request: Head https://172.17.1.201/v2/library/alpine/manifests/latest: x509: certificate signed by unknown authority

I have already setup 172.17.1.201 as an insecure registry of containerd, and restarted containerd.

    [plugins.cri.registry]
      [plugins.cri.registry.mirrors]
        [plugins.cri.registry.mirrors."172.17.1.201"]
          endpoint = ["http://172.17.1.201"]
        [plugins.cri.registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]

Steps to reproduce the issue:

  1. Push an image into Harbor
  2. Pull the image from Harbor

Describe the results you received:
Failed to pull image from Harbor.

Describe the results you expected:
Successfully pull image from Harbor.

Output of containerd --version:

containerd containerd.io 1.2.6 894b81a4b802e4eb2a91d1ce216b8817763c29fb

Any other relevant information:

$ crictl info 
{
  "status": {
    "conditions": [
      {
        "type": "RuntimeReady",
        "status": true,
        "reason": "",
        "message": ""
      },
      {
        "type": "NetworkReady",
        "status": true,
        "reason": "",
        "message": ""
      }
    ]
  },
  "config": {
    "containerd": {
      "snapshotter": "overlayfs",
      "defaultRuntime": {
        "runtimeType": "io.containerd.runtime.v1.linux",
        "runtimeEngine": "",
        "runtimeRoot": "",
        "options": null
      },
      "untrustedWorkloadRuntime": {
        "runtimeType": "",
        "runtimeEngine": "",
        "runtimeRoot": "",
        "options": null
      },
      "runtimes": null,
      "noPivot": false
    },
    "cni": {
      "binDir": "/opt/cni/bin",
      "confDir": "/etc/cni/net.d",
      "confTemplate": ""
    },
    "registry": {
      "mirrors": {
        "172.17.1.201": {
          "endpoint": [
            "http://172.17.1.201"
          ]
        },
        "docker.io": {
          "endpoint": [
            "https://registry-1.docker.io"
          ]
        }
      },
      "auths": null
    },
    "streamServerAddress": "127.0.0.1",
    "streamServerPort": "0",
    "enableSelinux": false,
    "sandboxImage": "k8s.gcr.io/pause:3.1",
    "statsCollectPeriod": 10,
    "systemdCgroup": false,
    "enableTLSStreaming": false,
    "x509KeyPairStreaming": {
      "tlsCertFile": "",
      "tlsKeyFile": ""
    },
    "maxContainerLogSize": 16384,
    "containerdRootDir": "/var/lib/containerd",
    "containerdEndpoint": "/run/containerd/containerd.sock",
    "rootDir": "/var/lib/containerd/io.containerd.grpc.v1.cri",
    "stateDir": "/run/containerd/io.containerd.grpc.v1.cri"
  },
  "golang": "go1.11.8"
}

$ crictl version 
Version:  0.1.0
RuntimeName:  containerd
RuntimeVersion:  1.2.6
RuntimeApiVersion:  v1alpha2

Activity

qianzhangxa
added
kind/bug
on Nov 25, 2019
fuweid

fuweid commented on Nov 25, 2019

@fuweid
fuweid
on Nov 25, 2019 · edited by fuweid
Member

@qianzhangxa thanks for reporting. Could you show your whole containerd configuration, please? And could you retry it with upgrading to last version of containerd? Thanks

qianzhangxa

qianzhangxa commented on Nov 25, 2019

@qianzhangxa
qianzhangxa
on Nov 25, 2019 · edited by qianzhangxa
Author

@fuweid Here is the whole containerd configuration:

$ cat /etc/containerd/config.toml
root = "/var/lib/containerd"
state = "/run/containerd"
oom_score = 0

[grpc]
  address = "/run/containerd/containerd.sock"
  uid = 0
  gid = 0
  max_recv_message_size = 16777216
  max_send_message_size = 16777216

[debug]
  address = ""
  uid = 0
  gid = 0
  level = ""

[metrics]
  # Set the metrics TCP address
  address = "172.17.0.2:1338"
  grpc_histogram = false

[cgroup]
  path = ""

[plugins]
  [plugins.cgroups]
    no_prometheus = false
  [plugins.cri]
    stream_server_address = "127.0.0.1"
    stream_server_port = "0"
    enable_selinux = false
    sandbox_image = "k8s.gcr.io/pause:3.1"
    stats_collect_period = 10
    systemd_cgroup = false
    enable_tls_streaming = false
    max_container_log_line_size = 16384
    [plugins.cri.containerd]
      snapshotter = "overlayfs"
      no_pivot = false
      [plugins.cri.containerd.default_runtime]
        runtime_type = "io.containerd.runtime.v1.linux"
        runtime_engine = ""
        runtime_root = ""
      [plugins.cri.containerd.untrusted_workload_runtime]
        runtime_type = ""
        runtime_engine = ""
        runtime_root = ""
    [plugins.cri.cni]
      bin_dir = "/opt/cni/bin"
      conf_dir = "/etc/cni/net.d"
      conf_template = ""
    [plugins.cri.registry]
      [plugins.cri.registry.mirrors]
        [plugins.cri.registry.mirrors."172.17.1.201"]
          endpoint = ["http://172.17.1.201"]
        [plugins.cri.registry.mirrors."docker.io"]
          endpoint = ["https://registry-1.docker.io"]
    [plugins.cri.x509_key_pair_streaming]
      tls_cert_file = ""
      tls_key_file = ""
  [plugins.diff-service]
    default = ["walking"]
  [plugins.linux]
    shim = "containerd-shim"
    runtime = "runc"
    runtime_root = ""
    no_shim = false
    shim_debug = false
  [plugins.opt]
    path = "/opt/containerd"
  [plugins.restart]
    interval = "10s"
  [plugins.scheduler]
    pause_threshold = 0.02
    deletion_threshold = 0
    mutation_threshold = 100
    schedule_delay = "0s"
    startup_delay = "100ms"

Do you mean there is no such issue with the latest version of containerd?

qianzhangxa

qianzhangxa commented on Nov 25, 2019

@qianzhangxa
qianzhangxa
on Nov 25, 2019 · edited by qianzhangxa
Author

@nustiueudinastea I think they are different, what you are trying to pull from is a secure registry (https), right? But my issue is about insecure registry (http).

nustiueudinastea

nustiueudinastea commented on Nov 25, 2019

@nustiueudinastea
nustiueudinastea
on Nov 25, 2019 · edited by nustiueudinastea

It was totally my fault, so I deleted my previous comment to not confuse other people. Problem was that containerd did not have access to the root certificates.

fuweid

fuweid commented on Nov 25, 2019

@fuweid
fuweid
on Nov 25, 2019
Member

@qianzhangxa it seems your registry has certificates and cri-containerd will check the certificate presented by the server. You can retry it with adding certificate in your client side.

qianzhangxa

qianzhangxa commented on Nov 25, 2019

@qianzhangxa
qianzhangxa
on Nov 25, 2019
Author

@fuweid What I want to try is the insecure registry feature of containerd, that's why I did not add Harbor's certificate to containerd.

fuweid

fuweid commented on Nov 25, 2019

@fuweid
fuweid
on Nov 25, 2019
Member

ping @Random-Liu , @mikebrow and @dmcgowan

it is ok to set http.Client InsecureSkipVerify to true if mirror endpoint's scheme is http?
https://github.com/containerd/cri/blob/0dcaf6e98719b02ad9a1cf93aa3c7dcb4225f7fc/pkg/server/image_pull.go#L313

cc @qianzhangxa

dmcgowan

dmcgowan commented on Nov 25, 2019

@dmcgowan
dmcgowan
on Nov 25, 2019
Member

it is ok to set http.Client InsecureSkipVerify to true if mirror endpoint's scheme is http?

no, this should be an explicit configuration

Random-Liu

Random-Liu commented on Nov 26, 2019

@Random-Liu
Random-Liu
on Nov 26, 2019 · edited by Random-Liu
Member

@fuweid @dmcgowan We can add an option explicitly for InsecureSkipVerify.

Here is Docker's doc for insecure-registries:

With insecure registries enabled, Docker goes through the following steps:

First, try using HTTPS.
If HTTPS is available but the certificate is invalid, ignore the error about the certificate.
If HTTPS is not available, fall back to HTTP.

Not sure whether we want that.

qianzhangxa

qianzhangxa commented on Nov 26, 2019

@qianzhangxa
qianzhangxa
on Nov 26, 2019 · edited by qianzhangxa
Author

@fuweid @dmcgowan @Random-Liu So containerd does not support insecure registry yet? I just followed the instructions here: https://github.com/containerd/cri/blob/master/docs/registry.md#configure-registry-endpoint, and it clearly describes an example for insecure registry:

  [plugins.cri.registry.mirrors."test.insecure-registry.io"]
    endpoint = ["http://HostIP2:Port2"]

So such insecure registry configuration in containerd actually cannot work as expected?

Random-Liu

Random-Liu commented on Nov 26, 2019

@Random-Liu
Random-Liu
on Nov 26, 2019 via email
Member
Random-Liu
mentioned this on Nov 26, 2019
Random-Liu

Random-Liu commented on Nov 26, 2019

@Random-Liu
Random-Liu
on Nov 26, 2019
Member

The fix: containerd/cri#1345

Random-Liu
closed this as completedin #1345on Nov 26, 2019

4 remaining items

belegent

belegent commented on Dec 9, 2019

@belegent
belegent
on Dec 9, 2019

The containerd has been already updated to v1.3.2.

xuguozhen

xuguozhen commented on Dec 18, 2019

@xuguozhen
xuguozhen
on Dec 18, 2019

@fuweid @dmcgowan @Random-Liu So containerd does not support insecure registry yet? I just followed the instructions here: https://github.com/containerd/cri/blob/master/docs/registry.md#configure-registry-endpoint, and it clearly describes an example for insecure registry:

  [plugins.cri.registry.mirrors."test.insecure-registry.io"]
    endpoint = ["http://HostIP2:Port2"]

So such insecure registry configuration in containerd actually cannot work as expected?

@qianzhangxa
Have your issue been resolved?
If so, what is the solution?

rtiwary-dev
mentioned this on Mar 9, 2020
howardjohn
mentioned this on Jun 6, 2020
alexellis
mentioned this on Aug 30, 2020
vsai9594

vsai9594 commented on Sep 7, 2020

@vsai9594
vsai9594
on Sep 7, 2020

Hi, i am facing similar issue. I added harbor as insecure registry in registries.conf , i am able to pull the images if i am using docker pull command but when i use the same image in kubernetes yaml file .. i am getting this "Failed to pull image "harbor.x.x.x.com/test/test-image:v1": rpc error: code = Unknown desc = failed to resolve image "harbor.x.x.x.com/test/test-image:v1": no available registry endpoint: failed to do request: Head https://harbor.x.x.x.com/v2/test/test-image/manifests/v1: x509: certificate signed by unknown authority"

kamileon
mentioned this on Dec 17, 2020
boeboe

boeboe commented on Feb 22, 2022

@boeboe
boeboe
on Feb 22, 2022 · edited by boeboe

This worked for me FYI

[plugins.cri.registry.mirrors."harbor.example.com"]
  endpoint = ["https://harbor.example.com"]
[plugins.cri.registry.configs."harbor.example.com".tls]
  insecure_skip_verify = true

I changed "io.containerd.grpc.v1.cri" to cri instead, and then it worked.

ddelange
mentioned this on Mar 30, 2022
drummerglen

drummerglen commented on May 19, 2022

@drummerglen
drummerglen
on May 19, 2022 · edited by drummerglen

This worked for me FYI

[plugins.cri.registry.mirrors."harbor.example.com"]
  endpoint = ["https://harbor.example.com"]
[plugins.cri.registry.configs."harbor.example.com".tls]
  insecure_skip_verify = true

I changed "io.containerd.grpc.v1.cri" to cri instead, and then it worked.

@boeboe Is't this parameter?
plugins."io.containerd.grpc.v1.cri".containerd
default_runtime_name = "cri"

amirad1

amirad1 commented on Sep 1, 2022

@amirad1
amirad1
on Sep 1, 2022

This worked for me FYI

[plugins.cri.registry.mirrors."harbor.example.com"]
  endpoint = ["https://harbor.example.com"]
[plugins.cri.registry.configs."harbor.example.com".tls]
  insecure_skip_verify = true

I changed "io.containerd.grpc.v1.cri" to cri instead, and then it worked.

@boeboe Is't this parameter? plugins."io.containerd.grpc.v1.cri".containerd default_runtime_name = "cri"

I was looking for this problem for several days, but this worked for me, thanks !

tataue

tataue commented on Nov 11, 2022

@tataue
tataue
on Nov 11, 2022

This worked for me FYI

[plugins.cri.registry.mirrors."harbor.example.com"]
  endpoint = ["https://harbor.example.com"]
[plugins.cri.registry.configs."harbor.example.com".tls]
  insecure_skip_verify = true

I changed "io.containerd.grpc.v1.cri" to cri instead, and then it worked.

not work for me containerd just restart failed:
containerd: failed to load TOML from /etc/containerd/config.toml: invalid plugin key URI "cri" expect io.containerd.x.vx

manuh-L

manuh-L commented on Jun 1, 2023

@manuh-L
manuh-L
on Jun 1, 2023

dev-control-plane containerd[6762]: containerd: failed to load TOML from /etc/containerd/config.toml: invalid plugin key URI "cri" expect io.containerd.x.vx

manuh-L

manuh-L commented on Jun 1, 2023

@manuh-L
manuh-L
on Jun 1, 2023 · edited by manuh-L

cat /etc/containerd/config.toml
version = 2

[plugins]
[plugins."io.containerd.grpc.v1.cri"]
restrict_oom_score_adj = false
sandbox_image = "registry.k8s.io/pause:3.7"
tolerate_missing_hugepages_controller = true
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
discard_unpacked_layers = true
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = "/etc/containerd/cri-base.json"
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
base_runtime_spec = "/etc/containerd/cri-base.json"
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.lab.com"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.lab.com".tls]
insecure_skip_verify = true
cert_file = "/etc/docker/certs.d/harbor.lab.com/harbor.lab.com.cert"
key_file = "/etc/docker/certs.d/harbor.lab.com/harbor.lab.com.key"
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
endpoint = ["https://registry.k8s.io", "https://k8s.gcr.io"]

[proxy_plugins]
[proxy_plugins.fuse-overlayfs]
address = "/run/containerd-fuse-overlayfs.sock"
type = "snapshot"

omegavlg2

omegavlg2 commented on May 8, 2024

@omegavlg2
omegavlg2
on May 8, 2024 · edited by omegavlg2

Hello everyone!

Could you please help me with the following containerd configuration? Here are the details:

containerd containerd.io 1.6.22 8165fea

cat /etc/containerd/config.toml
oom_score = -999

[debug]
level = "info"

[plugins.linux]
shim = "/home/kubernetes/bin/containerd-shim"
runtime = "/home/kubernetes/bin/runc"

[plugins.cri]
stream_server_address = "127.0.0.1"
enable_tls_streaming = false
sandbox_image = "cr.yandex/crpsjg1coh47p81vh2lc/pause:3.9"
[plugins.cri.containerd]
snapshotter = "overlayfs"

[plugins.cri.cni]
bin_dir = "/home/kubernetes/cni/bin"
conf_dir = "/etc/cni/net.d"

conf_template = "/etc/cni/cni.template"

[plugins."io.containerd.grpc.v1.cri".registry]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."10.71.0.6:8083"]
endpoint = ["http://10.71.0.6:8083"]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.configs."10.71.0.6:8083".tls]
insecure_skip_verify = true

ctr --debug image pull 10.71.0.6:8083/my-image:8.11.3
DEBU[0000] fetching image="10.71.0.6:8083/my-image:8.11.3"
DEBU[0000] resolving host="10.71.0.6:8083"
DEBU[0000] do request host="10.71.0.6:8083" request.header.accept="application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, /" request.header.user-agent=containerd/1.6.22 request.method=HEAD url="https://10.71.0.6:8083/v2/my-image/manifests/8.11.3"
INFO[0000] trying next host error="failed to do request: Head "https://10.71.0.6:8083/v2/my-image/manifests/8.11.3\": http: server gave HTTP response to HTTPS client" host="10.71.0.6:8083"
ctr: failed to resolve reference "10.71.0.6:8083/my-image:8.11.3": failed to do request: Head "https://10.71.0.6:8083/v2/my-image/manifests/8.11.3": http: server gave HTTP response to HTTPS client

Additionally, if you execute the docker pull command with insecure-registries enabled, everything works fine.

Thank you in advance for your assistance!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    kind/bug

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

      Development

      Participants

      @dmcgowan@boeboe@nustiueudinastea@tataue@Random-Liu

      Issue actions
        Containerd cannot pull image from insecure registry · Issue #3847 · containerd/containerd