As @slashzero@hakbox.social said, if you're using an additional nginx server, your docker nginx can't listen for port 80 or 443. Here's my host nginx reverse proxy's ssl section for reference:

server {
    server_name kek.henlo.fi;

        location / {
	proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection "upgrade";

        include proxy_params;
        proxy_pass http://localhost:9001;
    }

    listen 443 ssl http2;
    listen [::]:443 ssl http2;
    ssl_certificate /etc/letsencrypt/live/kek.henlo.fi/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/kek.henlo.fi/privkey.pem; # managed by Certbot
    include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
    ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
}

Maybe you're missing some proxy headers, or the docker compose isn't forwarding the correct ports.

But it can definitely be something completely different.