mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-10-16 12:17:34 +09:00
import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo
This commit is contained in:
Roy Tam
commit
dcd9973243
150858 changed files with 23884658 additions and 0 deletions
280
testing/mozharness/external_tools/clobberer.py
Normal file
280
testing/mozharness/external_tools/clobberer.py
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
#!/usr/bin/python
|
||||
# vim:sts=2 sw=2
|
||||
import sys
|
||||
import shutil
|
||||
import urllib2
|
||||
import urllib
|
||||
import os
|
||||
import traceback
|
||||
import time
|
||||
if os.name == 'nt':
|
||||
from win32file import RemoveDirectory, DeleteFile, \
|
||||
GetFileAttributesW, SetFileAttributesW, \
|
||||
FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY
|
||||
from win32api import FindFiles
|
||||
|
||||
clobber_suffix = '.deleteme'
|
||||
|
||||
|
||||
def ts_to_str(ts):
|
||||
if ts is None:
|
||||
return None
|
||||
return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))
|
||||
|
||||
|
||||
def write_file(ts, fn):
|
||||
assert isinstance(ts, int)
|
||||
f = open(fn, "w")
|
||||
f.write(str(ts))
|
||||
f.close()
|
||||
|
||||
|
||||
def read_file(fn):
|
||||
if not os.path.exists(fn):
|
||||
return None
|
||||
|
||||
data = open(fn).read().strip()
|
||||
try:
|
||||
return int(data)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def rmdirRecursiveWindows(dir):
|
||||
"""Windows-specific version of rmdirRecursive that handles
|
||||
path lengths longer than MAX_PATH.
|
||||
"""
|
||||
|
||||
dir = os.path.realpath(dir)
|
||||
# Make sure directory is writable
|
||||
SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL)
|
||||
|
||||
for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'):
|
||||
file_attr = ffrec[0]
|
||||
name = ffrec[8]
|
||||
if name == '.' or name == '..':
|
||||
continue
|
||||
full_name = os.path.join(dir, name)
|
||||
|
||||
if file_attr & FILE_ATTRIBUTE_DIRECTORY:
|
||||
rmdirRecursiveWindows(full_name)
|
||||
else:
|
||||
SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL)
|
||||
DeleteFile('\\\\?\\' + full_name)
|
||||
RemoveDirectory('\\\\?\\' + dir)
|
||||
|
||||
def rmdirRecursive(dir):
|
||||
"""This is a replacement for shutil.rmtree that works better under
|
||||
windows. Thanks to Bear at the OSAF for the code.
|
||||
(Borrowed from buildbot.slave.commands)"""
|
||||
if os.name == 'nt':
|
||||
rmdirRecursiveWindows(dir)
|
||||
return
|
||||
|
||||
if not os.path.exists(dir):
|
||||
# This handles broken links
|
||||
if os.path.islink(dir):
|
||||
os.remove(dir)
|
||||
return
|
||||
|
||||
if os.path.islink(dir):
|
||||
os.remove(dir)
|
||||
return
|
||||
|
||||
# Verify the directory is read/write/execute for the current user
|
||||
os.chmod(dir, 0700)
|
||||
|
||||
for name in os.listdir(dir):
|
||||
full_name = os.path.join(dir, name)
|
||||
# on Windows, if we don't have write permission we can't remove
|
||||
# the file/directory either, so turn that on
|
||||
if os.name == 'nt':
|
||||
if not os.access(full_name, os.W_OK):
|
||||
# I think this is now redundant, but I don't have an NT
|
||||
# machine to test on, so I'm going to leave it in place
|
||||
# -warner
|
||||
os.chmod(full_name, 0600)
|
||||
|
||||
if os.path.isdir(full_name):
|
||||
rmdirRecursive(full_name)
|
||||
else:
|
||||
# Don't try to chmod links
|
||||
if not os.path.islink(full_name):
|
||||
os.chmod(full_name, 0700)
|
||||
os.remove(full_name)
|
||||
os.rmdir(dir)
|
||||
|
||||
|
||||
def do_clobber(dir, dryrun=False, skip=None):
|
||||
try:
|
||||
for f in os.listdir(dir):
|
||||
if skip is not None and f in skip:
|
||||
print "Skipping", f
|
||||
continue
|
||||
clobber_path = f + clobber_suffix
|
||||
if os.path.isfile(f):
|
||||
print "Removing", f
|
||||
if not dryrun:
|
||||
if os.path.exists(clobber_path):
|
||||
os.unlink(clobber_path)
|
||||
# Prevent repeated moving.
|
||||
if f.endswith(clobber_suffix):
|
||||
os.unlink(f)
|
||||
else:
|
||||
shutil.move(f, clobber_path)
|
||||
os.unlink(clobber_path)
|
||||
elif os.path.isdir(f):
|
||||
print "Removing %s/" % f
|
||||
if not dryrun:
|
||||
if os.path.exists(clobber_path):
|
||||
rmdirRecursive(clobber_path)
|
||||
# Prevent repeated moving.
|
||||
if f.endswith(clobber_suffix):
|
||||
rmdirRecursive(f)
|
||||
else:
|
||||
shutil.move(f, clobber_path)
|
||||
rmdirRecursive(clobber_path)
|
||||
except:
|
||||
print "Couldn't clobber properly, bailing out."
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def getClobberDates(clobberURL, branch, buildername, builddir, slave, master):
|
||||
params = dict(branch=branch, buildername=buildername,
|
||||
builddir=builddir, slave=slave, master=master)
|
||||
url = "%s?%s" % (clobberURL, urllib.urlencode(params))
|
||||
print "Checking clobber URL: %s" % url
|
||||
# The timeout arg was added to urlopen() at Python 2.6
|
||||
# Deprecate this test when esr17 reaches EOL
|
||||
if sys.version_info[:2] < (2, 6):
|
||||
data = urllib2.urlopen(url).read().strip()
|
||||
else:
|
||||
data = urllib2.urlopen(url, timeout=30).read().strip()
|
||||
|
||||
retval = {}
|
||||
try:
|
||||
for line in data.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
builddir, builder_time, who = line.split(":")
|
||||
builder_time = int(builder_time)
|
||||
retval[builddir] = (builder_time, who)
|
||||
return retval
|
||||
except ValueError:
|
||||
print "Error parsing response from server"
|
||||
print data
|
||||
raise
|
||||
|
||||
if __name__ == "__main__":
|
||||
from optparse import OptionParser
|
||||
parser = OptionParser(
|
||||
"%prog [options] clobberURL branch buildername builddir slave master")
|
||||
parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
|
||||
default=False, help="don't actually delete anything")
|
||||
parser.add_option("-t", "--periodic", dest="period", type="float",
|
||||
default=None, help="hours between periodic clobbers")
|
||||
parser.add_option('-s', '--skip', help='do not delete this file/directory',
|
||||
action='append', dest='skip', default=['last-clobber'])
|
||||
parser.add_option('-d', '--dir', help='clobber this directory',
|
||||
dest='dir', default='.', type='string')
|
||||
parser.add_option('-v', '--verbose', help='be more verbose',
|
||||
dest='verbose', action='store_true', default=False)
|
||||
|
||||
options, args = parser.parse_args()
|
||||
if len(args) != 6:
|
||||
parser.error("Incorrect number of arguments")
|
||||
|
||||
if options.period:
|
||||
periodicClobberTime = options.period * 3600
|
||||
else:
|
||||
periodicClobberTime = None
|
||||
|
||||
clobberURL, branch, builder, my_builddir, slave, master = args
|
||||
|
||||
try:
|
||||
server_clobber_dates = getClobberDates(
|
||||
clobberURL, branch, builder, my_builddir, slave, master)
|
||||
except:
|
||||
if options.verbose:
|
||||
traceback.print_exc()
|
||||
print "Error contacting server"
|
||||
sys.exit(1)
|
||||
|
||||
if options.verbose:
|
||||
print "Server gave us", server_clobber_dates
|
||||
|
||||
now = int(time.time())
|
||||
|
||||
# Add ourself to the server_clobber_dates if it's not set
|
||||
# This happens when this slave has never been clobbered
|
||||
if my_builddir not in server_clobber_dates:
|
||||
server_clobber_dates[my_builddir] = None, ""
|
||||
|
||||
root_dir = os.path.abspath(options.dir)
|
||||
|
||||
for builddir, (server_clobber_date, who) in server_clobber_dates.items():
|
||||
builder_dir = os.path.join(root_dir, builddir)
|
||||
if not os.path.isdir(builder_dir):
|
||||
print "%s doesn't exist, skipping" % builder_dir
|
||||
continue
|
||||
os.chdir(builder_dir)
|
||||
|
||||
our_clobber_date = read_file("last-clobber")
|
||||
|
||||
clobber = False
|
||||
clobberType = None
|
||||
|
||||
print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date)
|
||||
print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date)
|
||||
|
||||
# If we don't have a last clobber date, then this is probably a fresh build.
|
||||
# We should only do a forced server clobber if we know when our last clobber
|
||||
# was, and if the server date is more recent than that.
|
||||
if server_clobber_date is not None and our_clobber_date is not None:
|
||||
# If the server is giving us a clobber date, compare the server's idea of
|
||||
# the clobber date to our last clobber date
|
||||
if server_clobber_date > our_clobber_date:
|
||||
# If the server's clobber date is greater than our last clobber date,
|
||||
# then we should clobber.
|
||||
clobber = True
|
||||
clobberType = "forced"
|
||||
# We should also update our clobber date to match the server's
|
||||
our_clobber_date = server_clobber_date
|
||||
if who:
|
||||
print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who)
|
||||
else:
|
||||
print "%s:Server is forcing a clobber" % builddir
|
||||
|
||||
if not clobber:
|
||||
# Disable periodic clobbers for builders that aren't my_builddir
|
||||
if builddir != my_builddir:
|
||||
continue
|
||||
|
||||
# Next, check if more than the periodicClobberTime period has passed since
|
||||
# our last clobber
|
||||
if our_clobber_date is None:
|
||||
# We've never been clobbered
|
||||
# Set our last clobber time to now, so that we'll clobber
|
||||
# properly after periodicClobberTime
|
||||
clobberType = "purged"
|
||||
our_clobber_date = now
|
||||
write_file(our_clobber_date, "last-clobber")
|
||||
elif periodicClobberTime and now > our_clobber_date + periodicClobberTime:
|
||||
# periodicClobberTime has passed since our last clobber
|
||||
clobber = True
|
||||
clobberType = "periodic"
|
||||
# Update our clobber date to now
|
||||
our_clobber_date = now
|
||||
print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime)
|
||||
|
||||
if clobber:
|
||||
# Finally, perform a clobber if we're supposed to
|
||||
print "%s:Clobbering..." % builddir
|
||||
do_clobber(builder_dir, options.dryrun, options.skip)
|
||||
write_file(our_clobber_date, "last-clobber")
|
||||
|
||||
# If this is the build dir for the current job, display the clobber type in TBPL.
|
||||
# Note in the case of purged clobber, we output the clobber type even though no
|
||||
# clobber was performed this time.
|
||||
if clobberType and builddir == my_builddir:
|
||||
print "TinderboxPrint: %s clobber" % clobberType
|
||||
Loading…
Add table
Add a link
Reference in a new issue