replace free_tls_certificates with certbot
authorJoshua Tauberer <jt@occams.info>
Sun, 13 May 2018 00:02:25 +0000 (20:02 -0400)
committerJoshua Tauberer <jt@occams.info>
Fri, 29 Jun 2018 20:46:21 +0000 (16:46 -0400)
CHANGELOG.md
conf/nginx.conf
management/daemon.py
management/daily_tasks.sh
management/ssl_certificates.py
management/templates/ssl.html
management/web_update.py
setup/management.sh
setup/migrate.py
setup/start.sh
setup/system.sh

index 63eaf47..9e4f2e0 100644 (file)
@@ -4,8 +4,14 @@ CHANGELOG
 v0.27 (June 14, 2018)
 ---------------------
 
+Mail:
+
 * A report of box activity, including sent/received mail totals and logins by user, is now emailed to the box's administrator user each week.
 * Update Roundcube to version 1.3.6 and Z-Push to version 2.3.9.
+
+Control Panel:
+
+* We now use EFF's `certbot` tool to provision HTTPS certificates instead of our home-grown free_tls_certificates package.
 * The undocumented feature for proxying web requests to another server now sets X-Forwarded-For.
 
 v0.26c (February 13, 2018)
index 0a08439..ce66275 100644 (file)
@@ -25,7 +25,7 @@ server {
                # This path must be served over HTTP for ACME domain validation.
                # We map this to a special path where our TLS cert provisioning
                # tool knows to store challenge response files.
-               alias $STORAGE_ROOT/ssl/lets_encrypt/acme_challenges/;
+               alias $STORAGE_ROOT/ssl/lets_encrypt/webroot/.well-known/acme-challenge/;
        }
 }
 
index f5ea215..2e23c8a 100755 (executable)
@@ -333,11 +333,16 @@ def ssl_get_status():
        from web_update import get_web_domains_info, get_web_domains
 
        # What domains can we provision certificates for? What unexpected problems do we have?
-       provision, cant_provision = get_certificates_to_provision(env, show_extended_problems=False)
+       provision, cant_provision = get_certificates_to_provision(env, show_valid_certs=False)
        
        # What's the current status of TLS certificates on all of the domain?
        domains_status = get_web_domains_info(env)
-       domains_status = [{ "domain": d["domain"], "status": d["ssl_certificate"][0], "text": d["ssl_certificate"][1] } for d in domains_status ]
+       domains_status = [
+               {
+                       "domain": d["domain"],
+                       "status": d["ssl_certificate"][0],
+                       "text": d["ssl_certificate"][1] + ((" " + cant_provision[d["domain"]] if d["domain"] in cant_provision else ""))
+               } for d in domains_status ]
 
        # Warn the user about domain names not hosted here because of other settings.
        for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
@@ -349,7 +354,6 @@ def ssl_get_status():
 
        return json_response({
                "can_provision": utils.sort_domains(provision, env),
-               "cant_provision": [{ "domain": domain, "problem": cant_provision[domain] } for domain in utils.sort_domains(cant_provision, env) ],
                "status": domains_status,
        })
 
@@ -376,11 +380,8 @@ def ssl_install_cert():
 @authorized_personnel_only
 def ssl_provision_certs():
        from ssl_certificates import provision_certificates
-       agree_to_tos_url = request.form.get('agree_to_tos_url')
-       status = provision_certificates(env,
-               agree_to_tos_url=agree_to_tos_url,
-               jsonable=True)
-       return json_response(status)
+       requests = provision_certificates(env, limit_domains=None)
+       return json_response({ "requests": requests })
 
 
 # WEB
index b5b628c..3054dd3 100755 (executable)
@@ -19,7 +19,7 @@ fi
 management/backup.py | management/email_administrator.py "Backup Status"
 
 # Provision any new certificates for new domains or domains with expiring certificates.
-management/ssl_certificates.py -q --headless | management/email_administrator.py "Error Provisioning TLS Certificate"
+management/ssl_certificates.py -q | management/email_administrator.py "Error Provisioning TLS Certificate"
 
 # Run status checks and email the administrator if anything changed.
 management/status_checks.py --show-changes | management/email_administrator.py "Status Checks Change Notice"
index 303571b..200a346 100755 (executable)
@@ -1,7 +1,7 @@
 #!/usr/local/lib/mailinabox/env/bin/python
 # Utilities for installing and selecting SSL certificates.
 
-import os, os.path, re, shutil
+import os, os.path, re, shutil, subprocess, tempfile
 
 from utils import shell, safe_domain_name, sort_domains
 import idna
@@ -24,6 +24,16 @@ def get_ssl_certificates(env):
                if not os.path.exists(ssl_root):
                        return
                for fn in os.listdir(ssl_root):
+                       if fn == 'ssl_certificate.pem':
+                               # This is always a symbolic link
+                               # to the certificate to use for
+                               # PRIMARY_HOSTNAME. Don't let it
+                               # be eligible for use because we
+                               # could end up creating a symlink
+                               # to itself --- we want to find
+                               # the cert that it should be a
+                               # symlink to.
+                               continue
                        fn = os.path.join(ssl_root, fn)
                        if os.path.isfile(fn):
                                yield fn
@@ -74,6 +84,12 @@ def get_ssl_certificates(env):
 
                # Add this cert to the list of certs usable for the domains.
                for domain in cert_domains:
+                       # The primary hostname can only use a certificate mapped
+                       # to the system private key.
+                       if domain == env['PRIMARY_HOSTNAME']:
+                               if cert._private_key._filename != os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem'):
+                                       continue
+
                        domains.setdefault(domain, []).append(cert)
 
        # Sort the certificates to prefer good ones.
@@ -81,6 +97,7 @@ def get_ssl_certificates(env):
        now = datetime.datetime.utcnow()
        ret = { }
        for domain, cert_list in domains.items():
+               #for c in cert_list: print(domain, c.not_valid_before, c.not_valid_after, "("+str(now)+")", c.issuer, c.subject, c._filename)
                cert_list.sort(key = lambda cert : (
                        # must be valid NOW
                        cert.not_valid_before <= now <= cert.not_valid_after,
@@ -124,21 +141,22 @@ def get_ssl_certificates(env):
 
        return ret
 
-def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, raw=False):
-       # Get the system certificate info.
-       ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem'))
-       ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
-       system_certificate = {
-               "private-key": ssl_private_key,
-               "certificate": ssl_certificate,
-               "primary-domain": env['PRIMARY_HOSTNAME'],
-               "certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]),
-       }
-
-       if domain == env['PRIMARY_HOSTNAME']:
-               # The primary domain must use the server certificate because
-               # it is hard-coded in some service configuration files.
-               return system_certificate
+def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False, use_main_cert=True):
+       if use_main_cert:
+               # Get the system certificate info.
+               ssl_private_key = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_private_key.pem'))
+               ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
+               system_certificate = {
+                       "private-key": ssl_private_key,
+                       "certificate": ssl_certificate,
+                       "primary-domain": env['PRIMARY_HOSTNAME'],
+                       "certificate_object": load_pem(load_cert_chain(ssl_certificate)[0]),
+               }
+
+               if domain == env['PRIMARY_HOSTNAME']:
+                       # The primary domain must use the server certificate because
+                       # it is hard-coded in some service configuration files.
+                       return system_certificate
 
        wildcard_domain = re.sub("^[^\.]+", "*", domain)
        if domain in ssl_certificates:
@@ -155,112 +173,97 @@ def get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=False
 
 # PROVISIONING CERTIFICATES FROM LETSENCRYPT
 
-def get_certificates_to_provision(env, show_extended_problems=True, force_domains=None):
-       # Get a set of domain names that we should now provision certificates
-       # for. Provision if a domain name has no valid certificate or if any
-       # certificate is expiring in 14 days. If provisioning anything, also
-       # provision certificates expiring within 30 days. The period between
-       # 14 and 30 days allows us to consolidate domains into multi-domain
-       # certificates for domains expiring around the same time.
+def get_certificates_to_provision(env, limit_domains=None, show_valid_certs=True):
+       # Get a set of domain names that we can provision certificates for
+       # using certbot. We start with domains that the box is serving web
+       # for and subtract:
+       # * domains not in limit_domains if limit_domains is not empty
+       # * domains with custom "A" records, i.e. they are hosted elsewhere
+       # * domains with actual "A" records that point elsewhere
+       # * domains that already have certificates that will be valid for a while
 
        from web_update import get_web_domains
+       from status_checks import query_dns, normalize_ip
 
-       import datetime
-       now = datetime.datetime.utcnow()
-
-       # Get domains with missing & expiring certificates.
-       certs = get_ssl_certificates(env)
-       domains = set()
-       domains_if_any = set()
-       problems = { }
-       for domain in get_web_domains(env):
-               # If the user really wants a cert for certain domains, include it.
-               if force_domains:
-                       if force_domains == "ALL" or (isinstance(force_domains, list) and domain in force_domains):
-                               domains.add(domain)
-                       continue
-
-               # Include this domain if its certificate is missing, self-signed, or expiring soon.
-               try:
-                       cert = get_domain_ssl_files(domain, certs, env, allow_missing_cert=True)
-               except FileNotFoundError as e:
-                       # system certificate is not present
-                       problems[domain] = "Error: " + str(e)
-                       continue
-               if cert is None:
-                       # No valid certificate available.
-                       domains.add(domain)
-               else:
-                       cert = cert["certificate_object"]
-                       if cert.issuer == cert.subject:
-                               # This is self-signed. Get a real one.
-                               domains.add(domain)
-                       
-                       # Valid certificate today, but is it expiring soon?
-                       elif cert.not_valid_after-now < datetime.timedelta(days=14):
-                               domains.add(domain)
-                       elif cert.not_valid_after-now < datetime.timedelta(days=30):
-                               domains_if_any.add(domain)
-
-                       # It's valid. Should we report its validness?
-                       elif show_extended_problems:
-                               problems[domain] = "The certificate is valid for at least another 30 days --- no need to replace."
+       existing_certs = get_ssl_certificates(env)
 
-       # Warn the user about domains hosted elsewhere.
-       if not force_domains and show_extended_problems:
-               for domain in set(get_web_domains(env, exclude_dns_elsewhere=False)) - set(get_web_domains(env)):
-                       problems[domain] = "The domain's DNS is pointed elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
+       plausible_web_domains = get_web_domains(env, exclude_dns_elsewhere=False)
+       actual_web_domains = get_web_domains(env)
 
-       # Filter out domains that we can't provision a certificate for.
-       def can_provision_for_domain(domain):
-               from status_checks import query_dns, normalize_ip
+       domains_to_provision = set()
+       domains_cant_provision = { }
 
-               # Does the domain resolve to this machine in public DNS? If not,
-               # we can't do domain control validation. For IPv6 is configured,
-               # make sure both IPv4 and IPv6 are correct because we don't know
-               # how Let's Encrypt will connect.
-               for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]:
-                       if not value: continue # IPv6 is not configured
-                       response = query_dns(domain, rtype)
-                       if response != normalize_ip(value):
-                               problems[domain] = "The domain name does not resolve to this machine: DNS %s resolved to %s." % (rtype, response)
-                               return False
-
-               return True
-
-       domains = set(filter(can_provision_for_domain, domains))
-
-       # If there are any domains we definitely will provision for, add in
-       # additional domains to do at this time.
-       if len(domains) > 0:
-               domains |= set(filter(can_provision_for_domain, domains_if_any))
-
-       return (domains, problems)
+       for domain in plausible_web_domains:
+               # Skip domains that the user doesn't want to provision now.
+               if limit_domains and domain not in limit_domains:
+                       continue
 
-def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extended_problems=True, force_domains=None, jsonable=False):
-       import requests.exceptions
-       import acme.messages
+               # Check that there isn't an explicit A/AAAA record.
+               if domain not in actual_web_domains:
+                       domains_cant_provision[domain] = "The domain has a custom DNS A/AAAA record that points the domain elsewhere, so there is no point to installing a TLS certificate here and we could not automatically provision one anyway because provisioning requires access to the website (which isn't here)."
 
-       from free_tls_certificates import client
+               # Check that the DNS resolves to here.
+               else:
 
+                       # Does the domain resolve to this machine in public DNS? If not,
+                       # we can't do domain control validation. For IPv6 is configured,
+                       # make sure both IPv4 and IPv6 are correct because we don't know
+                       # how Let's Encrypt will connect.
+                       bad_dns = []
+                       for rtype, value in [("A", env["PUBLIC_IP"]), ("AAAA", env.get("PUBLIC_IPV6"))]:
+                               if not value: continue # IPv6 is not configured
+                               response = query_dns(domain, rtype)
+                               if response != normalize_ip(value):
+                                       bad_dns.append("%s (%s)" % (response, rtype))
+       
+                       if bad_dns:
+                               domains_cant_provision[domain] = "The domain name does not resolve to this machine: " \
+                                       + (", ".join(bad_dns)) \
+                                       + "."
+                       
+                       else:
+                               # DNS is all good.
+
+                               # Check for a good existing cert.
+                               existing_cert = get_domain_ssl_files(domain, existing_certs, env, use_main_cert=False)
+                               if existing_cert:
+                                       existing_cert_check = check_certificate(domain, existing_cert['certificate'], existing_cert['private-key'],
+                                               warn_if_expiring_soon=14)
+                                       if existing_cert_check[0] == "OK":
+                                               if show_valid_certs:
+                                                       domains_cant_provision[domain] = "The domain has a valid certificate already. ({} Certificate: {}, private key {})".format(
+                                                               existing_cert_check[1],
+                                                               existing_cert['certificate'],
+                                                               existing_cert['private-key'])
+                                               continue
+
+                               domains_to_provision.add(domain)
+
+       return (domains_to_provision, domains_cant_provision)
+
+def provision_certificates(env, limit_domains):
        # What domains should we provision certificates for? And what
        # errors prevent provisioning for other domains.
-       domains, problems = get_certificates_to_provision(env, force_domains=force_domains, show_extended_problems=show_extended_problems)
+       domains, domains_cant_provision = get_certificates_to_provision(env, limit_domains=limit_domains)
+
+       # Build a list of what happened on each domain or domain-set.
+       ret = []
+       for domain, error in domains_cant_provision.items():
+               ret.append({
+                       "domains": [domain],
+                       "log": [error],
+                       "result": "skipped",
+               })
 
-       # Exit fast if there is nothing to do.
-       if len(domains) == 0:
-               return {
-                       "requests": [],
-                       "problems": problems,
-               }
 
        # Break into groups of up to 100 certificates at a time, which is Let's Encrypt's
        # limit for a single certificate. We'll sort to put related domains together.
+       max_domains_per_group = 100
        domains = sort_domains(domains, env)
        certs = []
        while len(domains) > 0:
-               certs.append( domains[0:100] )
-               domains = domains[100:]
+               certs.append( domains[:max_domains_per_group] )
+               domains = domains[max_domains_per_group:]
 
        # Prepare to provision.
 
@@ -269,115 +272,82 @@ def provision_certificates(env, agree_to_tos_url=None, logger=None, show_extende
        if not os.path.exists(account_path):
                os.mkdir(account_path)
 
-       # Where should we put ACME challenge files. This is mapped to /.well-known/acme_challenge
-       # by the nginx configuration.
-       challenges_path = os.path.join(account_path, 'acme_challenges')
-       if not os.path.exists(challenges_path):
-               os.mkdir(challenges_path)
-
-       # Read in the private key that we use for all TLS certificates. We'll need that
-       # to generate a CSR (done by free_tls_certificates).
-       with open(os.path.join(env['STORAGE_ROOT'], 'ssl/ssl_private_key.pem'), 'rb') as f:
-               private_key = f.read()
-
        # Provision certificates.
-
-       ret = []
        for domain_list in certs:
-               # For return.
-               ret_item = {
+               ret.append({
                        "domains": domain_list,
                        "log": [],
-               }
-               ret.append(ret_item)
-
-               # Logging for free_tls_certificates.
-               def my_logger(message):
-                       if logger: logger(message)
-                       ret_item["log"].append(message)
-
-               # Attempt to provision a certificate.
+               })
                try:
-                       try:
-                               cert = client.issue_certificate(
-                                       domain_list,
-                                       account_path,
-                                       agree_to_tos_url=agree_to_tos_url,
-                                       private_key=private_key,
-                                       logger=my_logger)
-
-                       except client.NeedToTakeAction as e:
-                               # Write out the ACME challenge files.
-                               for action in e.actions:
-                                       if isinstance(action, client.NeedToInstallFile):
-                                               fn = os.path.join(challenges_path, action.file_name)
-                                               with open(fn, 'w') as f:
-                                                       f.write(action.contents)
-                                       else:
-                                               raise ValueError(str(action))
-
-                               # Try to provision now that the challenge files are installed.
-
-                               cert = client.issue_certificate(
-                                       domain_list,
-                                       account_path,
-                                       private_key=private_key,
-                                       logger=my_logger)
-
-               except client.NeedToAgreeToTOS as e:
-                       # The user must agree to the Let's Encrypt terms of service agreement
-                       # before any further action can be taken.
-                       ret_item.update({
-                               "result": "agree-to-tos",
-                               "url": e.url,
-                       })
-
-               except client.WaitABit as e:
-                       # We need to hold on for a bit before querying again to see if we can
-                       # acquire a provisioned certificate.
-                       import time, datetime
-                       ret_item.update({
-                               "result": "wait",
-                               "until": e.until_when if not jsonable else e.until_when.isoformat(),
-                               "seconds": (e.until_when - datetime.datetime.now()).total_seconds()
-                       })
-
-               except client.AccountDataIsCorrupt as e:
-                       # This is an extremely rare condition.
-                       ret_item.update({
-                               "result": "error",
-                               "message": "Something unexpected went wrong. It looks like your local Let's Encrypt account data is corrupted. There was a problem with the file " + e.account_file_path + ".",
-                       })
-
-               except (client.InvalidDomainName, client.NeedToTakeAction, client.ChallengeFailed, client.RateLimited, acme.messages.Error, requests.exceptions.RequestException) as e:
-                       ret_item.update({
-                               "result": "error",
-                               "message": "Something unexpected went wrong: " + str(e),
-                       })
-
-               else:
-                       # A certificate was issued.
-
-                       install_status = install_cert(domain_list[0], cert['cert'].decode("ascii"), b"\n".join(cert['chain']).decode("ascii"), env, raw=True)
-
-                       # str indicates the certificate was not installed.
-                       if isinstance(install_status, str):
-                               ret_item.update({
-                                       "result": "error",
-                                       "message": "Something unexpected was wrong with the provisioned certificate: " + install_status,
-                               })
-                       else:
-                               # A list indicates success and what happened next.
-                               ret_item["log"].extend(install_status)
-                               ret_item.update({
-                                       "result": "installed",
-                               })
+                       # Create a CSR file for our master private key so that certbot
+                       # uses our private key.
+                       key_file = os.path.join(env['STORAGE_ROOT'], 'ssl', 'ssl_private_key.pem')
+                       with tempfile.NamedTemporaryFile() as csr_file:
+                               # We could use openssl, but certbot requires
+                               # that the CN domain and SAN domains match
+                               # the domain list passed to certbot, and adding
+                               # SAN domains openssl req is ridiculously complicated.
+                               # subprocess.check_output([
+                               #       "openssl", "req", "-new",
+                               #       "-key", key_file,
+                               #       "-out", csr_file.name,
+                               #       "-subj", "/CN=" + domain_list[0],
+                               #       "-sha256" ])
+                               from cryptography import x509
+                               from cryptography.hazmat.backends import default_backend
+                               from cryptography.hazmat.primitives.serialization import Encoding
+                               from cryptography.hazmat.primitives import hashes
+                               from cryptography.x509.oid import NameOID
+                               builder = x509.CertificateSigningRequestBuilder()
+                               builder = builder.subject_name(x509.Name([ x509.NameAttribute(NameOID.COMMON_NAME, domain_list[0]) ]))
+                               builder = builder.add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)
+                               builder = builder.add_extension(x509.SubjectAlternativeName(
+                                       [x509.DNSName(d) for d in domain_list]
+                               ), critical=False)
+                               request = builder.sign(load_pem(load_cert_chain(key_file)[0]), hashes.SHA256(), default_backend())
+                               with open(csr_file.name, "wb") as f:
+                                       f.write(request.public_bytes(Encoding.PEM))
+
+                               # Provision, writing to a temporary file.
+                               webroot = os.path.join(account_path, 'webroot')
+                               os.makedirs(webroot, exist_ok=True)
+                               with tempfile.TemporaryDirectory() as d:
+                                       cert_file = os.path.join(d, 'cert_and_chain.pem')
+                                       print("Provisioning TLS certificates for " + ", ".join(domain_list) + ".")
+                                       certbotret = subprocess.check_output([
+                                               "certbot",
+                                               "certonly",
+                                               #"-v", # just enough to see ACME errors
+                                               "--non-interactive", # will fail if user hasn't registered during Mail-in-a-Box setup
+
+                                               "-d", ",".join(domain_list), # first will be main domain
+
+                                               "--csr", csr_file.name, # use our private key; unfortunately this doesn't work with auto-renew so we need to save cert manually
+                                               "--cert-path", os.path.join(d, 'cert'), # we only use the full chain
+                                               "--chain-path", os.path.join(d, 'chain'), # we only use the full chain
+                                               "--fullchain-path", cert_file,
+
+                                               "--webroot", "--webroot-path", webroot,
+
+                                               "--config-dir", account_path,
+                                               #"--staging",
+                                       ], stderr=subprocess.STDOUT).decode("utf8")
+                                       install_cert_copy_file(cert_file, env)
+
+                       ret[-1]["log"].append(certbotret)
+                       ret[-1]["result"] = "installed"
+               except subprocess.CalledProcessError as e:
+                       ret[-1]["log"].append(e.output.decode("utf8"))
+                       ret[-1]["result"] = "error"
+               except Exception as e:
+                       ret[-1]["log"].append(str(e))
+                       ret[-1]["result"] = "error"
+
+       # Run post-install steps.
+       ret.extend(post_install_func(env))
 
        # Return what happened with each certificate request.
-       return {
-               "requests": ret,
-               "problems": problems,
-       }
+       return ret
 
 def provision_certificates_cmdline():
        import sys
@@ -388,151 +358,39 @@ def provision_certificates_cmdline():
        Lock(die=True).forever()
        env = load_environment()
 
-       verbose = False
-       headless = False
-       force_domains = None
-       show_extended_problems = True
-       
-       args = list(sys.argv)
-       args.pop(0) # program name
-       if args and args[0] == "-v":
-               verbose = True
-               args.pop(0)
-       if args and args[0] == "-q":
-               show_extended_problems = False
-               args.pop(0)
-       if args and args[0] == "--headless":
-               headless = True
-               args.pop(0)
-       if args and args[0] == "--force":
-               force_domains = "ALL"
-               args.pop(0)
-       else:
-               force_domains = args
-
-       agree_to_tos_url = None
-       while True:
-               # Run the provisioning script. This installs certificates. If there are
-               # a very large number of domains on this box, it issues separate
-               # certificates for groups of domains. We have to check the result for
-               # each group.
-               def my_logger(message):
-                       if verbose:
-                               print(">", message)
-               status = provision_certificates(env, agree_to_tos_url=agree_to_tos_url, logger=my_logger, force_domains=force_domains, show_extended_problems=show_extended_problems)
-               agree_to_tos_url = None # reset to prevent infinite looping
-
-               if not status["requests"]:
-                       # No domains need certificates.
-                       if not headless or verbose:
-                               if len(status["problems"]) == 0:
-                                       print("No domains hosted on this box need a new TLS certificate at this time.")
-                               elif len(status["problems"]) > 0:
-                                       print("No TLS certificates could be provisoned at this time:")
-                                       print()
-                                       for domain in sort_domains(status["problems"], env):
-                                               print("%s: %s" % (domain, status["problems"][domain]))
-
-                       sys.exit(0)
-
-               # What happened?
-               wait_until = None
-               wait_domains = []
-               for request in status["requests"]:
-                       if request["result"] == "agree-to-tos":
-                               # We may have asked already in a previous iteration.
-                               if agree_to_tos_url is not None:
-                                       continue
-
-                               # Can't ask the user a question in this mode. Warn the user that something
-                               # needs to be done.
-                               if headless:
-                                       print(", ".join(request["domains"]) + " need a new or renewed TLS certificate.")
-                                       print()
-                                       print("This box can't do that automatically for you until you agree to Let's Encrypt's")
-                                       print("Terms of Service agreement. Use the Mail-in-a-Box control panel to provision")
-                                       print("certificates for these domains.")
-                                       sys.exit(1)
-
-                               print("""
-I'm going to provision a TLS certificate (formerly called a SSL certificate)
-for you from Let's Encrypt (letsencrypt.org).
-
-TLS certificates are cryptographic keys that ensure communication between
-you and this box are secure when getting and sending mail and visiting
-websites hosted on this box. Let's Encrypt is a free provider of TLS
-certificates.
-
-Please open this document in your web browser:
+       quiet = False
+       domains = []
 
-%s
+       for arg in sys.argv[1:]:
+               if arg == "-q":
+                       quiet = True
+               else:
+                       domains.append(arg)
 
-It is Let's Encrypt's terms of service agreement. If you agree, I can
-provision that TLS certificate. If you don't agree, you will have an
-opportunity to install your own TLS certificate from the Mail-in-a-Box
-control panel.
+       # Go.
+       status = provision_certificates(env, limit_domains=domains)
 
-Do you agree to the agreement? Type Y or N and press <ENTER>: """
-                                % request["url"], end='', flush=True)
-                       
-                               if sys.stdin.readline().strip().upper() != "Y":
-                                       print("\nYou didn't agree. Quitting.")
-                                       sys.exit(1)
-
-                               # Okay, indicate agreement on next iteration.
-                               agree_to_tos_url = request["url"]
-
-                       if request["result"] == "wait":
-                               # Must wait. We'll record until when. The wait occurs below.
-                               if wait_until is None:
-                                       wait_until = request["until"]
-                               else:
-                                       wait_until = max(wait_until, request["until"])
-                               wait_domains += request["domains"]
-
-                       if request["result"] == "error":
-                               print(", ".join(request["domains"]) + ":")
-                               print(request["message"])
-
-                       if request["result"] == "installed":
-                               print("A TLS certificate was successfully installed for " + ", ".join(request["domains"]) + ".")
-
-               if wait_until:
-                       # Wait, then loop.
-                       import time, datetime
+       # Show what happened.
+       for request in status:
+               if isinstance(request, str):
+                       print(request)
+               else:
+                       if quiet and request['result'] == 'skipped':
+                               continue
+                       print(request['result'] + ":", ", ".join(request['domains']) + ":")
+                       for line in request["log"]:
+                               print(line)
                        print()
-                       print("A TLS certificate was requested for: " + ", ".join(wait_domains) + ".")
-                       first = True
-                       while wait_until > datetime.datetime.now():
-                               if not headless or first:
-                                       print ("We have to wait", int(round((wait_until - datetime.datetime.now()).total_seconds())), "seconds for the certificate to be issued...")
-                               time.sleep(10)
-                               first = False
-
-                       continue # Loop!
-
-               if agree_to_tos_url:
-                       # The user agrees to the TOS. Loop to try again by agreeing.
-                       continue # Loop!
-
-               # Unless we were instructed to wait, or we just agreed to the TOS,
-               # we're done for now.
-               break
-
-       # And finally show the domains with problems.
-       if len(status["problems"]) > 0:
-               print("TLS certificates could not be provisoned for:")
-               for domain in sort_domains(status["problems"], env):
-                       print("%s: %s" % (domain, status["problems"][domain]))
+
 
 # INSTALLING A NEW CERTIFICATE FROM THE CONTROL PANEL
 
 def create_csr(domain, ssl_key, country_code, env):
        return shell("check_output", [
-                "openssl", "req", "-new",
-                "-key", ssl_key,
-                "-sha256",
-                "-subj", "/C=%s/CN=%s" % (country_code, domain)])
+                               "openssl", "req", "-new",
+                               "-key", ssl_key,
+                               "-sha256",
+                               "-subj", "/C=%s/CN=%s" % (country_code, domain)])
 
 def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
        # Write the combined cert+chain to a temporary path and validate that it is OK.
@@ -553,6 +411,16 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
                        cert_status += " " + cert_status_details
                return cert_status
 
+       # Copy certifiate into ssl directory.
+       install_cert_copy_file(fn, env)
+
+       # Run post-install steps.
+       ret = post_install_func(env)
+       if raw: return ret
+       return "\n".join(ret)
+
+
+def install_cert_copy_file(fn, env):
        # Where to put it?
        # Make a unique path for the certificate.
        from cryptography.hazmat.primitives import hashes
@@ -570,14 +438,26 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
        os.makedirs(os.path.dirname(ssl_certificate), exist_ok=True)
        shutil.move(fn, ssl_certificate)
 
-       ret = ["OK"]
 
-       # When updating the cert for PRIMARY_HOSTNAME, symlink it from the system
+def post_install_func(env):
+       ret = []
+
+       # Get the certificate to use for PRIMARY_HOSTNAME.
+       ssl_certificates = get_ssl_certificates(env)
+       cert = get_domain_ssl_files(env['PRIMARY_HOSTNAME'], ssl_certificates, env, use_main_cert=False)
+       if not cert:
+               # Ruh-row, we don't have any certificate usable
+               # for the primary hostname.
+               ret.append("there is no valid certificate for " + env['PRIMARY_HOSTNAME'])
+
+       # Symlink the best cert for PRIMARY_HOSTNAME to the system
        # certificate path, which is hard-coded for various purposes, and then
        # restart postfix and dovecot.
-       if domain == env['PRIMARY_HOSTNAME']:
+       system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
+       if cert and os.readlink(system_ssl_certificate) != cert['certificate']:
                # Update symlink.
-               system_ssl_certificate = os.path.join(os.path.join(env["STORAGE_ROOT"], 'ssl', 'ssl_certificate.pem'))
+               ret.append("updating primary certificate")
+               ssl_certificate = cert['certificate']
                os.unlink(system_ssl_certificate)
                os.symlink(ssl_certificate, system_ssl_certificate)
 
@@ -593,12 +473,12 @@ def install_cert(domain, ssl_cert, ssl_chain, env, raw=False):
        # Update the web configuration so nginx picks up the new certificate file.
        from web_update import do_web_update
        ret.append( do_web_update(env) )
-       if raw: return ret
-       return "\n".join(ret)
+
+       return ret
 
 # VALIDATION OF CERTIFICATES
 
-def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=True, rounded_time=False, just_check_domain=False):
+def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring_soon=10, rounded_time=False, just_check_domain=False):
        # Check that the ssl_certificate & ssl_private_key files are good
        # for the provided domain.
 
@@ -704,7 +584,7 @@ def check_certificate(domain, ssl_certificate, ssl_private_key, warn_if_expiring
                        # We'll renew it with Lets Encrypt.
                        expiry_info = "The certificate expires on %s." % cert_expiration_date.strftime("%x")
 
-               if ndays <= 10 and warn_if_expiring_soon:
+               if warn_if_expiring_soon and ndays <= warn_if_expiring_soon:
                        # Warn on day 10 to give 4 days for us to automatically renew the
                        # certificate, which occurs on day 14.
                        return ("The certificate is expiring soon: " + expiry_info, None)
index 54b2a94..a6b913e 100644 (file)
@@ -8,7 +8,7 @@
 <p>You need a TLS certificate for this box&rsquo;s hostname ({{hostname}}) and every other domain name and subdomain that this box is hosting a website for (see the list below).</p>
 
 <div id="ssl_provision">
-<h3>Provision a certificate</h3>
+<h3>Provision certificates</h3>
 
 <div id="ssl_provision_p" style="display: none; margin-top: 1.5em">
   <button onclick='return provision_tls_cert();' class='btn btn-primary' style="float: left; margin: 0 1.5em 1em 0;">Provision</button>
 <div class="clearfix"> </div>
 
 <div id="ssl_provision_result"></div>
-
-<div id="ssl_provision_problems_div" style="display: none;">
-<p style="margin-bottom: .5em;">Certificates cannot be automatically provisioned for:</p>
-<table id="ssl_provision_problems" style="margin-top: 0;" class="table">
-  <thead>
-    <tr>
-      <th>Domain</th>
-      <th>Problem</th>
-    </tr>
-  </thead>
-  <tbody>
-  </tbody>
-</table>
-<p>Use the <em>Install Certificate</em> button below for these domains.</p>
-</div>
 </div>
 
 <h3>Certificate status</h3>
@@ -103,24 +88,12 @@ function show_tls(keep_provisioning_shown) {
       // provisioning status
 
       if (!keep_provisioning_shown)
-        $('#ssl_provision').toggle(res.can_provision.length + res.cant_provision.length > 0)
+        $('#ssl_provision').toggle(res.can_provision.length > 0)
 
       $('#ssl_provision_p').toggle(res.can_provision.length > 0);
       if (res.can_provision.length > 0)
           $('#ssl_provision_p span').text(res.can_provision.join(", "));
 
-      $('#ssl_provision_problems_div').toggle(res.cant_provision.length > 0);
-      $('#ssl_provision_problems tbody').text("");
-      for (var i = 0; i < res.cant_provision.length; i++) {
-        var domain = res.cant_provision[i];
-        var row = $("<tr><th class='domain'><a href=''></a></th><td class='status'></td></tr>");
-        $('#ssl_provision_problems tbody').append(row);
-        row.attr('data-domain', domain.domain);
-        row.find('.domain a').text(domain.domain);
-        row.find('.domain a').attr('href', 'https://' + domain.domain);
-        row.find('.status').text(domain.problem);
-      }
-
       // certificate status
       var domains = res.status;
       var tb = $('#ssl_domains tbody');
@@ -196,20 +169,15 @@ function install_cert() {
     });
 }
 
-var agree_to_tos_url_prompt = null;
-var agree_to_tos_url = null;
 function provision_tls_cert() {
   // Automatically provision any certs.
   $('#ssl_provision_p .btn').attr('disabled', '1'); // prevent double-clicks
   api(
     "/ssl/provision",
     "POST",
-    {
-      agree_to_tos_url: agree_to_tos_url
-    },
+    { },
     function(status) {
       // Clear last attempt.
-      agree_to_tos_url = null;
       $('#ssl_provision_result').text("");
       may_reenable_provision_button = true;
 
@@ -225,52 +193,33 @@ function provision_tls_cert() {
       for (var i = 0; i < status.requests.length; i++) {
         var r = status.requests[i];
 
+        if (r.result == "skipped") {
+          // not interested --- this domain wasn't in the table
+          // to begin with
+          continue;
+        }
+
         // create an HTML block to display the results of this request
         var n = $("<div><h4/><p/></div>");
         $('#ssl_provision_result').append(n);
 
+        // plain log line
+        if (typeof r === "string") {
+          n.find("p").text(r);
+          continue;
+        }
+
         // show a header only to disambiguate request blocks
         if (status.requests.length > 0)
           n.find("h4").text(r.domains.join(", "));
 
-        if (r.result == "agree-to-tos") {
-          // user needs to agree to Let's Encrypt's TOS
-          agree_to_tos_url_prompt = r.url;
-          $('#ssl_provision_p .btn').attr('disabled', '1');
-          n.find("p").html("Please open and review <a href='" + r.url + "' target='_blank'>Let's Encrypt's terms of service agreement</a>. You must agree to their terms for a certificate to be automatically provisioned from them.");
-          n.append($('<button onclick="agree_to_tos_url = agree_to_tos_url_prompt; return provision_tls_cert();" class="btn btn-success" style="margin-left: 2em">Agree &amp; Try Again</button>'));
-
-          // don't re-enable the Provision button -- user must use the Agree button
-          may_reenable_provision_button = false;
-
-        } else if (r.result == "error") {
+        if (r.result == "error") {
           n.find("p").addClass("text-danger").text(r.message);
 
-        } else if (r.result == "wait") {
-          // Show a button that counts down to zero, at which point it becomes enabled.
-          n.find("p").text("A certificate is now in the process of being provisioned, but it takes some time. Please wait until the Finish button is enabled, and then click it to acquire the certificate.");
-          var b = $('<button onclick="return provision_tls_cert();" class="btn btn-success" style="margin-left: 2em">Finish</button>');
-          b.attr("disabled", "1");
-          var now = new Date();
-          n.append(b);
-          function ready_to_finish() {
-            var remaining = Math.round(r.seconds - (new Date() - now)/1000);
-            if (remaining > 0) {
-              setTimeout(ready_to_finish, 1000);
-              b.text("Finish (" + remaining + "...)")
-            } else {
-              b.text("Finish (ready)")
-              b.removeAttr("disabled");
-            }
-          }
-          ready_to_finish();
-
-          // don't re-enable the Provision button -- user must use the Retry button when it becomes enabled
-          may_reenable_provision_button = false;
-
         } else if (r.result == "installed") {
           n.find("p").addClass("text-success").text("The TLS certificate was provisioned and installed.");
           setTimeout("show_tls(true)", 1); // update main table of certificate statuses, call with arg keep_provisioning_shown true so that we don't clear what we just outputted
+
         }
 
         // display the detailed log info in case of problems
@@ -278,7 +227,6 @@ function provision_tls_cert() {
         n.append(trace);
         for (var j = 0; j < r.log.length; j++)
           trace.append($("<div/>").text(r.log[j]));
-
       }
 
       if (may_reenable_provision_button)
index 1bd28e3..61b38a7 100644 (file)
@@ -201,8 +201,11 @@ def get_web_domains_info(env):
 
        # for the SSL config panel, get cert status
        def check_cert(domain):
-               tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True)
-               if tls_cert is None: return ("danger", "No Certificate Installed")
+               try:
+                       tls_cert = get_domain_ssl_files(domain, ssl_certificates, env, allow_missing_cert=True)
+               except OSError: # PRIMARY_HOSTNAME cert is missing
+                       tls_cert = None
+               if tls_cert is None: return ("danger", "No certificate installed.")
                cert_status, cert_status_details = check_certificate(domain, tls_cert["certificate"], tls_cert["private-key"])
                if cert_status == "OK":
                        return ("success", "Signed & valid. " + cert_status_details)
index 690ccb8..064906d 100755 (executable)
@@ -6,6 +6,18 @@ echo "Installing Mail-in-a-Box system management daemon..."
 
 # DEPENDENCIES
 
+# We used to install management daemon-related Python packages
+# directly to /usr/local/lib. We moved to a virtualenv because
+# these packages might conflict with apt-installed packages.
+# We may have a lingering version of acme that conflcits with
+# certbot, which we're about to install below, so remove it
+# first. Once acme is installed by an apt package, this might
+# break the package version and `apt-get install --reinstall python3-acme`
+# might be needed in that case.
+while [ -d /usr/local/lib/python3.4/dist-packages/acme ]; do
+       pip3 uninstall -y acme;
+done
+
 # duplicity is used to make backups of user data. It uses boto
 # (via Python 2) to do backups to AWS S3. boto from the Ubuntu
 # package manager is too out-of-date -- it doesn't support the newer
@@ -14,7 +26,10 @@ echo "Installing Mail-in-a-Box system management daemon..."
 #
 # python-virtualenv is used to isolate the Python 3 packages we
 # install via pip from the system-installed packages.
-apt_install duplicity python-pip python-virtualenv
+#
+# certbot installs EFF's certbot which we use to
+# provision free TLS certificates.
+apt_install duplicity python-pip python-virtualenv certbot
 hide_output pip2 install --upgrade boto
 
 # Create a virtualenv for the installation of Python 3 packages
@@ -32,13 +47,10 @@ hide_output $venv/bin/pip install --upgrade pip
 # Install other Python 3 packages used by the management daemon.
 # The first line is the packages that Josh maintains himself!
 # NOTE: email_validator is repeated in setup/questions.sh, so please keep the versions synced.
-# Force acme to be updated because it seems to need it after the
-# pip/setuptools breakage (see above) and the ACME protocol may
-# have changed (I got an error on one of my systems).
 hide_output $venv/bin/pip install --upgrade \
-       rtyaml "email_validator>=1.0.0" "free_tls_certificates>=0.1.3" "exclusiveprocess" \
+       rtyaml "email_validator>=1.0.0" "exclusiveprocess" \
        flask dnspython python-dateutil \
-       "idna>=2.0.0" "cryptography==2.2.2" "acme==0.20.0" boto psutil
+       "idna>=2.0.0" "cryptography==2.2.2" boto psutil
 
 # CONFIGURATION
 
index 45f748b..1d5911a 100755 (executable)
@@ -137,6 +137,17 @@ def migration_10(env):
                                shutil.move(sslcert, newname)
                                os.rmdir(d)
 
+def migration_11(env):
+       # Archive the old Let's Encrypt account directory managed by free_tls_certificates
+       # because we'll use that path now for the directory managed by certbot.
+       try:
+               old_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt')
+               new_path = os.path.join(env["STORAGE_ROOT"], 'ssl', 'lets_encrypt-old')
+               shutil.move(old_path, new_path)
+       except:
+               # meh
+               pass
+
 def get_current_migration():
        ver = 0
        while True:
index 0409647..86b34c8 100755 (executable)
@@ -127,13 +127,21 @@ tools/web_update
 # fail2ban was first configured, but they should exist now.
 restart_service fail2ban
 
-# If DNS is already working, try to provision TLS certficates from Let's Encrypt.
-# Suppress extra reasons why domains aren't getting a new certificate.
-management/ssl_certificates.py -q
-
 # If there aren't any mail users yet, create one.
 source setup/firstuser.sh
 
+# Register with Let's Encrypt, including agreeing to the Terms of Service. This
+# is an interactive command.
+if [ ! -d $STORAGE_ROOT/ssl/lets_encrypt/accounts/acme-v01.api.letsencrypt.org/ ]; then
+echo
+echo "-----------------------------------------------"
+echo "Mail-in-a-Box uses Let's Encrypt to provision free certificates"
+echo "to enable HTTPS connections to your box. You'll now be asked to agree"
+echo "to Let's Encrypt's terms of service."
+echo
+certbot register --register-unsafely-without-email --config-dir $STORAGE_ROOT/ssl/lets_encrypt
+fi
+
 # Done.
 echo
 echo "-----------------------------------------------"
index d26d4aa..0472805 100755 (executable)
@@ -68,17 +68,10 @@ then
        fi
 fi
 
-# ### Add Mail-in-a-Box's PPA.
-
-# We've built several .deb packages on our own that we want to include.
-# One is a replacement for Ubuntu's stock postgrey package that makes
-# some enhancements. The other is dovecot-lucene, a Lucene-based full
-# text search plugin for (and by) dovecot, which is not available in
-# Ubuntu currently.
-#
-# So, first ensure add-apt-repository is installed, then use it to install
-# the [mail-in-a-box ppa](https://launchpad.net/~mail-in-a-box/+archive/ubuntu/ppa).
+# ### Add PPAs.
 
+# We install some non-standard Ubuntu packages maintained by us and other
+# third-party providers. First ensure add-apt-repository is installed.
 
 if [ ! -f /usr/bin/add-apt-repository ]; then
        echo "Installing add-apt-repository..."
@@ -86,11 +79,21 @@ if [ ! -f /usr/bin/add-apt-repository ]; then
        apt_install software-properties-common
 fi
 
+# [Main-in-a-Box's own PPA](https://launchpad.net/~mail-in-a-box/+archive/ubuntu/ppa)
+# holds several .deb packages that we built on our own.
+# One is a replacement for Ubuntu's stock postgrey package that makes
+# some enhancements. The other is dovecot-lucene, a Lucene-based full
+# text search plugin for (and by) dovecot, which is not available in
+# Ubuntu currently.
+
 hide_output add-apt-repository -y ppa:mail-in-a-box/ppa
+hide_output add-apt-repository -y ppa:certbot/certbot
 
 # ### Update Packages
 
-# Update system packages to make sure we have the latest upstream versions of things from Ubuntu.
+# Update system packages to make sure we have the latest upstream versions
+# of things from Ubuntu, as well as the directory of packages provide by the
+# PPAs so we can install those packages later.
 
 echo Updating system packages...
 hide_output apt-get update