from __future__ import absolute_import
import contextlib
import json
import os
import random
import re
import socket
import ssl
import time
from mercurial.i18n import _
from mercurial.node import hex, nullid
from mercurial import (
commands,
configitems,
error,
exchange,
extensions,
hg,
match as matchmod,
pycompat,
registrar,
scmutil,
urllibcompat,
util,
vfs,
)
EXIT_PURGE_CACHE = 72
testedwith = (
b"4.5 4.6 4.7 4.8 4.9 5.0 5.1 5.2 5.3 5.4 5.5 5.6 5.7 5.8 5.9 6.0 6.1 6.2 6.3 6.4"
)
minimumhgversion = b"4.5"
cmdtable = {}
command = registrar.command(cmdtable)
configtable = {}
configitem = registrar.configitem(configtable)
configitem(b"robustcheckout", b"retryjittermin", default=configitems.dynamicdefault)
configitem(b"robustcheckout", b"retryjittermax", default=configitems.dynamicdefault)
def getsparse():
from mercurial import sparse
return sparse
def peerlookup(remote, v):
with remote.commandexecutor() as e:
return e.callcommand(b"lookup", {b"key": v}).result()
@command(
b"robustcheckout",
[
(b"", b"upstream", b"", b"URL of upstream repo to clone from"),
(b"r", b"revision", b"", b"Revision to check out"),
(b"b", b"branch", b"", b"Branch to check out"),
(b"", b"purge", False, b"Whether to purge the working directory"),
(b"", b"sharebase", b"", b"Directory where shared repos should be placed"),
(
b"",
b"networkattempts",
3,
b"Maximum number of attempts for network operations",
),
(b"", b"sparseprofile", b"", b"Sparse checkout profile to use (path in repo)"),
(
b"U",
b"noupdate",
False,
b"the clone will include an empty working directory\n"
b"(only a repository)",
),
],
b"[OPTION]... URL DEST",
norepo=True,
)
def robustcheckout(
ui,
url,
dest,
upstream=None,
revision=None,
branch=None,
purge=False,
sharebase=None,
networkattempts=None,
sparseprofile=None,
noupdate=False,
):
if not revision and not branch:
raise error.Abort(b"must specify one of --revision or --branch")
if revision and branch:
raise error.Abort(b"cannot specify both --revision and --branch")
if revision:
if (
len(revision) < 12
or len(revision) > 40
or not re.match(b"^[a-f0-9]+$", revision)
):
raise error.Abort(
b"--revision must be a SHA-1 fragment 12-40 characters long"
)
sharebase = sharebase or ui.config(b"share", b"pool")
if not sharebase:
raise error.Abort(
b"share base directory not defined; refusing to operate",
hint=b"define share.pool config option or pass --sharebase",
)
if sparseprofile:
try:
extensions.find(b"sparse")
except KeyError:
raise error.Abort(
b"sparse extension must be enabled to use --sparseprofile"
)
ui.warn(b"(using Mercurial %s)\n" % util.version())
ui.setconfig(b"worker", b"backgroundclose", False)
ui.setconfig(b"http", b"timeout", 600)
ui.setconfig(b"progress", b"delay", 1.0)
ui.setconfig(b"progress", b"refresh", 1.0)
ui.setconfig(b"progress", b"assume-tty", True)
sharebase = os.path.realpath(sharebase)
optimes = []
behaviors = set()
start = time.time()
try:
return _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattempts,
sparse_profile=sparseprofile,
noupdate=noupdate,
)
finally:
overall = time.time() - start
optimes.append(("overall", overall))
def record_op(name):
if "remove-store" in behaviors:
name += "_rmstore"
if "remove-wdir" in behaviors:
name += "_rmwdir"
optimes.append((name, overall))
if "clone" in behaviors and "create-store" in behaviors:
record_op("overall_clone")
if "sparse-update" in behaviors:
record_op("overall_clone_sparsecheckout")
else:
record_op("overall_clone_fullcheckout")
elif "pull" in behaviors or "clone" in behaviors:
record_op("overall_pull")
if "sparse-update" in behaviors:
record_op("overall_pull_sparsecheckout")
else:
record_op("overall_pull_fullcheckout")
if "empty-wdir" in behaviors:
record_op("overall_pull_emptywdir")
else:
record_op("overall_pull_populatedwdir")
else:
record_op("overall_nopull")
if "sparse-update" in behaviors:
record_op("overall_nopull_sparsecheckout")
else:
record_op("overall_nopull_fullcheckout")
if "empty-wdir" in behaviors:
record_op("overall_nopull_emptywdir")
else:
record_op("overall_nopull_populatedwdir")
server_url = urllibcompat.urlreq.urlparse(url).netloc
if "TASKCLUSTER_INSTANCE_TYPE" in os.environ:
perfherder = {
"framework": {
"name": "vcs",
},
"suites": [],
}
for op, duration in optimes:
perfherder["suites"].append(
{
"name": op,
"value": duration,
"lowerIsBetter": True,
"shouldAlert": False,
"serverUrl": server_url.decode("utf-8"),
"hgVersion": util.version().decode("utf-8"),
"extraOptions": [os.environ["TASKCLUSTER_INSTANCE_TYPE"]],
"subtests": [],
}
)
ui.write(
b"PERFHERDER_DATA: %s\n"
% pycompat.bytestr(json.dumps(perfherder, sort_keys=True))
)
def _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattemptlimit,
networkattempts=None,
sparse_profile=None,
noupdate=False,
):
if not networkattempts:
networkattempts = [1]
def callself():
return _docheckout(
ui,
url,
dest,
upstream,
revision,
branch,
purge,
sharebase,
optimes,
behaviors,
networkattemptlimit,
networkattempts=networkattempts,
sparse_profile=sparse_profile,
noupdate=noupdate,
)
@contextlib.contextmanager
def timeit(op, behavior):
behaviors.add(behavior)
errored = False
try:
start = time.time()
yield
except Exception:
errored = True
raise
finally:
elapsed = time.time() - start
if errored:
op += "_errored"
optimes.append((op, elapsed))
ui.write(b"ensuring %s@%s is available at %s\n" % (url, revision or branch, dest))
destvfs = vfs.vfs(dest, audit=False, realpath=True)
def deletesharedstore(path=None):
storepath = path or destvfs.read(b".hg/sharedpath").strip()
if storepath.endswith(b".hg"):
storepath = os.path.dirname(storepath)
storevfs = vfs.vfs(storepath, audit=False)
storevfs.rmtree(forcibly=True)
if destvfs.exists() and not destvfs.exists(b".hg"):
raise error.Abort(b"destination exists but no .hg directory")
if destvfs.exists() and sparse_profile and not destvfs.exists(b".hg/sparse"):
raise error.Abort(
b"cannot enable sparse profile on existing non-sparse checkout",
hint=b"use a separate working directory to use sparse",
)
if not sparse_profile and destvfs.exists(b".hg/sparse"):
raise error.Abort(
b"cannot use non-sparse checkout on existing sparse checkout",
hint=b"use a separate working directory to use sparse",
)
if destvfs.exists(b".hg") and not destvfs.exists(b".hg/sharedpath"):
ui.warn(b"(destination is not shared; deleting)\n")
with timeit("remove_unshared_dest", "remove-wdir"):
destvfs.rmtree(forcibly=True)
if destvfs.exists(b".hg/sharedpath"):
storepath = destvfs.read(b".hg/sharedpath").strip()
ui.write(b"(existing repository shared store: %s)\n" % storepath)
if not os.path.exists(storepath):
ui.warn(b"(shared store does not exist; deleting destination)\n")
with timeit("removed_missing_shared_store", "remove-wdir"):
destvfs.rmtree(forcibly=True)
elif not re.search(b"[a-f0-9]{40}/\\.hg$", storepath.replace(b"\\", b"/")):
ui.warn(
b"(shared store does not belong to pooled storage; "
b"deleting destination to improve efficiency)\n"
)
with timeit("remove_unpooled_store", "remove-wdir"):
destvfs.rmtree(forcibly=True)
if destvfs.isfileorlink(b".hg/wlock"):
ui.warn(
b"(dest has an active working directory lock; assuming it is "
b"left over from a previous process and that the destination "
b"is corrupt; deleting it just to be sure)\n"
)
with timeit("remove_locked_wdir", "remove-wdir"):
destvfs.rmtree(forcibly=True)
def handlerepoerror(e):
if pycompat.bytestr(e) == _(b"abandoned transaction found"):
ui.warn(b"(abandoned transaction found; trying to recover)\n")
repo = hg.repository(ui, dest)
if not repo.recover():
ui.warn(b"(could not recover repo state; deleting shared store)\n")
with timeit("remove_unrecovered_shared_store", "remove-store"):
deletesharedstore()
ui.warn(b"(attempting checkout from beginning)\n")
return callself()
raise
def handlenetworkfailure():
if networkattempts[0] >= networkattemptlimit:
raise error.Abort(
b"reached maximum number of network attempts; giving up\n"
)
ui.warn(
b"(retrying after network failure on attempt %d of %d)\n"
% (networkattempts[0], networkattemptlimit)
)
backoff = (2 ** networkattempts[0] - 1) * 1.5
jittermin = ui.configint(b"robustcheckout", b"retryjittermin", 1000)
jittermax = ui.configint(b"robustcheckout", b"retryjittermax", 5000)
backoff += float(random.randint(jittermin, jittermax)) / 1000.0
ui.warn(b"(waiting %.2fs before retry)\n" % backoff)
time.sleep(backoff)
networkattempts[0] += 1
def handlepullerror(e):
if isinstance(e, error.Abort):
if e.args[0] == _(b"repository is unrelated"):
ui.warn(b"(repository is unrelated; deleting)\n")
destvfs.rmtree(forcibly=True)
return True
elif e.args[0].startswith(_(b"stream ended unexpectedly")):
ui.warn(b"%s\n" % e.args[0])
handlenetworkfailure()
return True
elif isinstance(e, error.ResponseError):
if e.args[0].startswith(_(b"unexpected response from remote server:")):
ui.warn(b"(unexpected response from remote server; retrying)\n")
destvfs.rmtree(forcibly=True)
handlenetworkfailure()
return True
elif isinstance(e, ssl.SSLError):
ui.warn(b"ssl error: %s\n" % pycompat.bytestr(str(e)))
handlenetworkfailure()
return True
elif isinstance(e, urllibcompat.urlerr.httperror) and e.code >= 500:
ui.warn(b"http error: %s\n" % pycompat.bytestr(str(e.reason)))
handlenetworkfailure()
return True
elif isinstance(e, urllibcompat.urlerr.urlerror):
if isinstance(e.reason, socket.error):
ui.warn(b"socket error: %s\n" % pycompat.bytestr(str(e.reason)))
handlenetworkfailure()
return True
else:
ui.warn(
b"unhandled URLError; reason type: %s; value: %s\n"
% (
pycompat.bytestr(e.reason.__class__.__name__),
pycompat.bytestr(str(e.reason)),
)
)
elif isinstance(e, socket.timeout):
ui.warn(b"socket timeout\n")
handlenetworkfailure()
return True
else:
ui.warn(
b"unhandled exception during network operation; type: %s; "
b"value: %s\n"
% (pycompat.bytestr(e.__class__.__name__), pycompat.bytestr(str(e)))
)
return False
cloneurl = upstream or url
try:
clonepeer = hg.peer(ui, {}, cloneurl)
rootnode = peerlookup(clonepeer, b"0")
except error.RepoLookupError:
raise error.Abort(b"unable to resolve root revision from clone source")
except (
error.Abort,
ssl.SSLError,
urllibcompat.urlerr.urlerror,
socket.timeout,
) as e:
if handlepullerror(e):
return callself()
raise
if rootnode == nullid:
raise error.Abort(b"source repo appears to be empty")
storepath = os.path.join(sharebase, hex(rootnode))
storevfs = vfs.vfs(storepath, audit=False)
if storevfs.isfileorlink(b".hg/store/lock"):
ui.warn(
b"(shared store has an active lock; assuming it is left "
b"over from a previous process and that the store is "
b"corrupt; deleting store and destination just to be "
b"sure)\n"
)
if destvfs.exists():
with timeit("remove_dest_active_lock", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_active_lock", "remove-store"):
storevfs.rmtree(forcibly=True)
if storevfs.exists() and not storevfs.exists(b".hg/requires"):
ui.warn(
b"(shared store missing requires file; this is a really "
b"odd failure; deleting store and destination)\n"
)
if destvfs.exists():
with timeit("remove_dest_no_requires", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_no_requires", "remove-store"):
storevfs.rmtree(forcibly=True)
if storevfs.exists(b".hg/requires"):
requires = set(storevfs.read(b".hg/requires").splitlines())
if b"share-safe" in requires:
requires |= set(storevfs.read(b".hg/store/requires").splitlines())
required = {b"dotencode", b"fncache"}
missing = required - requires
if missing:
ui.warn(
b"(shared store missing requirements: %s; deleting "
b"store and destination to ensure optimal behavior)\n"
% b", ".join(sorted(missing))
)
if destvfs.exists():
with timeit("remove_dest_missing_requires", "remove-wdir"):
destvfs.rmtree(forcibly=True)
with timeit("remove_shared_store_missing_requires", "remove-store"):
storevfs.rmtree(forcibly=True)
created = False
if not destvfs.exists():
if util.safehasattr(util, "ensuredirs"):
makedirs = util.ensuredirs
else:
makedirs = util.makedirs
makedirs(os.path.dirname(destvfs.base), notindexed=True)
makedirs(sharebase, notindexed=True)
if upstream:
ui.write(b"(cloning from upstream repo %s)\n" % upstream)
if not storevfs.exists():
behaviors.add(b"create-store")
try:
with timeit("clone", "clone"):
shareopts = {b"pool": sharebase, b"mode": b"identity"}
res = hg.clone(
ui,
{},
clonepeer,
dest=dest,
update=False,
shareopts=shareopts,
stream=True,
)
except (
error.Abort,
ssl.SSLError,
urllibcompat.urlerr.urlerror,
socket.timeout,
) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn(b"(repo corruption: %s; deleting shared store)\n" % e)
with timeit("remove_shared_store_revlogerror", "remote-store"):
deletesharedstore()
return callself()
if res is None:
raise error.Abort(b"clone failed")
if not destvfs.exists(b".hg/sharedpath"):
raise error.Abort(b"clone did not create a shared repo")
created = True
repo = hg.repository(ui, dest)
havewantedrev = False
if revision:
try:
ctx = scmutil.revsingle(repo, revision)
except error.RepoLookupError:
ctx = None
if ctx:
if not ctx.hex().startswith(revision):
raise error.Abort(
b"--revision argument is ambiguous",
hint=b"must be the first 12+ characters of a SHA-1 fragment",
)
checkoutrevision = ctx.hex()
havewantedrev = True
if not havewantedrev:
ui.write(b"(pulling to obtain %s)\n" % (revision or branch,))
remote = None
try:
remote = hg.peer(repo, {}, url)
pullrevs = [peerlookup(remote, revision or branch)]
checkoutrevision = hex(pullrevs[0])
if branch:
ui.warn(
b"(remote resolved %s to %s; "
b"result is not deterministic)\n" % (branch, checkoutrevision)
)
if checkoutrevision in repo:
ui.warn(b"(revision already present locally; not pulling)\n")
else:
with timeit("pull", "pull"):
pullop = exchange.pull(repo, remote, heads=pullrevs)
if not pullop.rheads:
raise error.Abort(b"unable to pull requested revision")
except (
error.Abort,
ssl.SSLError,
urllibcompat.urlerr.urlerror,
socket.timeout,
) as e:
if handlepullerror(e):
return callself()
raise
except error.RepoError as e:
return handlerepoerror(e)
except error.RevlogError as e:
ui.warn(b"(repo corruption: %s; deleting shared store)\n" % e)
deletesharedstore()
return callself()
finally:
if remote:
remote.close()
if noupdate:
ui.write(b"(skipping update since `-U` was passed)\n")
return None
if purge and not created:
ui.write(b"(purging working directory)\n")
purge = getattr(commands, "purge", None)
if not purge:
purge = extensions.find(b"purge").purge
try:
old_sparse_fn = getattr(repo.dirstate, "_sparsematchfn", None)
if old_sparse_fn is not None:
repo.dirstate._sparsematchfn = lambda: matchmod.always()
with timeit("purge", "purge"):
if purge(
ui,
repo,
all=True,
abort_on_err=True,
**{"print": None, "print0": None, "dirs": None, "files": None}
):
raise error.Abort(b"error purging")
finally:
if old_sparse_fn is not None:
repo.dirstate._sparsematchfn = old_sparse_fn
if repo[b"."].node() == nullid:
behaviors.add("empty-wdir")
else:
behaviors.add("populated-wdir")
if sparse_profile:
sparsemod = getsparse()
try:
repo.filectx(sparse_profile, changeid=checkoutrevision).data()
except error.ManifestLookupError:
raise error.Abort(
b"sparse profile %s does not exist at revision "
b"%s" % (sparse_profile, checkoutrevision)
)
old_config = sparsemod.parseconfig(
repo.ui, repo.vfs.tryread(b"sparse"), b"sparse"
)
old_includes, old_excludes, old_profiles = old_config
if old_profiles == {sparse_profile} and not old_includes and not old_excludes:
ui.write(
b"(sparse profile %s already set; no need to update "
b"sparse config)\n" % sparse_profile
)
else:
if old_includes or old_excludes or old_profiles:
ui.write(
b"(replacing existing sparse config with profile "
b"%s)\n" % sparse_profile
)
else:
ui.write(b"(setting sparse config to profile %s)\n" % sparse_profile)
def parentchange(repo):
if util.safehasattr(repo.dirstate, "changing_parents"):
return repo.dirstate.changing_parents(repo)
return repo.dirstate.parentchange()
with repo.wlock(), parentchange(repo), timeit(
"sparse_update_config", "sparse-update-config"
):
fcounts = list(
map(
len,
sparsemod._updateconfigandrefreshwdir(
repo, [], [], [sparse_profile], force=True
),
)
)
repo.ui.status(
b"%d files added, %d files dropped, "
b"%d files conflicting\n" % tuple(fcounts)
)
ui.write(b"(sparse refresh complete)\n")
op = "update_sparse" if sparse_profile else "update"
behavior = "update-sparse" if sparse_profile else "update"
with timeit(op, behavior):
if commands.update(ui, repo, rev=checkoutrevision, clean=True):
raise error.Abort(b"error updating")
ui.write(b"updated to %s\n" % checkoutrevision)
return None
def extsetup(ui):
for ext in (b"purge", b"share"):
try:
extensions.find(ext)
except KeyError:
extensions.load(ui, ext, None)