wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-08 08:48:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

Issue #1859 - Part 4: Update mar file generation scripts for use of xz.

Browse source 
- Update packaging scripts to support both lzma and bzip2
- Update python script to support lzma
- Update test scripts to support lzma to be thorough

This all requires xz to be available in the path on the build system.
This commit is contained in:
Moonchild 2022-01-21 22:22:07 +00:00 • committed by roytam1
commit 121d56e082
9 changed files with 163 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

83
tools/update-packaging/make_incremental_updates.py
View file

@ -4,7 +4,7 @@
import os
import shutil
import sha
import hashlib
from os.path import join, getsize
from stat import *
import re
@ -12,9 +12,9 @@ import sys
import getopt
import time
import datetime
import bz2
import string
import tempfile
import io
class PatchInfo:
""" Represents the meta-data associated with a patch
@ -107,23 +107,23 @@ class PatchInfo:
""" Create the v2 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev2.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv2, '\n'))
manifest_file.writelines("\n")
manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv2).encode('ascii')))
manifest_file.writelines(io.BytesIO(b"\n"))
manifest_file.close()
bzip_file(manifest_file_path)
xz_file(manifest_file_path)
self.archive_files.append('"updatev2.manifest"')
""" Create the v3 manifest file in the root of the work_dir """
manifest_file_path = os.path.join(self.work_dir,"updatev3.manifest")
manifest_file = open(manifest_file_path, "wb")
manifest_file.writelines("type \"partial\"\n")
manifest_file.writelines(string.join(self.manifestv3, '\n'))
manifest_file.writelines("\n")
manifest_file.writelines(io.BytesIO(b"type \"partial\"\n"))
manifest_file.writelines(io.BytesIO('\n'.join(self.manifestv3).encode('ascii')))
manifest_file.writelines(io.BytesIO(b"\n"))
manifest_file.close()
bzip_file(manifest_file_path)
xz_file(manifest_file_path)
self.archive_files.append('"updatev3.manifest"')
def build_marfile_entry_hash(self, root_path):
@ -176,8 +176,8 @@ class MarFileEntry:
def calc_file_sha_digest(self, filename):
""" Returns sha digest of given filename"""
file_content = open(filename, 'r').read()
return sha.new(file_content).digest()
file_content = open(filename, 'rb').read()
return hashlib.sha1(file_content).digest()
def sha(self):
""" Returns sha digest of file repreesnted by this _marfile_entry"""
@ -188,7 +188,7 @@ class MarFileEntry:
def exec_shell_cmd(cmd):
"""Execs shell cmd and raises an exception if the cmd fails"""
if (os.system(cmd)):
raise Exception, "cmd failed "+cmd
raise Exception("cmd failed "+cmd)
def copy_file(src_file_abs_path, dst_file_abs_path):
@ -199,19 +199,19 @@ def copy_file(src_file_abs_path, dst_file_abs_path):
# Copy the file over
shutil.copy2(src_file_abs_path, dst_file_abs_path)
def bzip_file(filename):
""" Bzip's the file in place. The original file is replaced with a bzip'd version of itself
def xz_file(filename):
""" XZ compresses the file in place. The original file is replaced with the xz compressed version of itself
assumes the path is absolute"""
exec_shell_cmd('bzip2 -z9 "' + filename+'"')
os.rename(filename+".bz2",filename)
exec_shell_cmd('xz --compress --x86 --lzma2 --format=xz --check=crc64 "' + filename+'"')
os.rename(filename+".xz",filename)
def bunzip_file(filename):
""" Bzip's the file in palce. The original file is replaced with a bunzip'd version of itself.
doesn't matter if the filename ends in .bz2 or not"""
if not filename.endswith(".bz2"):
os.rename(filename, filename+".bz2")
filename=filename+".bz2"
exec_shell_cmd('bzip2 -d "' + filename+'"')
def xzunzip_file(filename):
""" xz decompresses the file in palce. The original file is replaced with a xz decompressed version of itself.
doesn't matter if the filename ends in .xz or not"""
if not filename.endswith(".xz"):
os.rename(filename, filename+".xz")
filename=filename+".xz"
exec_shell_cmd('xz -d "' + filename+'"')
def extract_mar(filename, work_dir):
@ -241,14 +241,14 @@ def create_partial_patch_for_file(from_marfile_entry, to_marfile_entry, shas, pa
if not os.path.exists(patch_file_dir):
os.makedirs(patch_file_dir)
# Create bzip'd patch file
# Create xz'd patch file
exec_shell_cmd("mbsdiff "+from_marfile_entry.abs_path+" "+to_marfile_entry.abs_path+" "+patch_file_abs_path)
bzip_file(patch_file_abs_path)
xz_file(patch_file_abs_path)
# Create bzip's full file
# Create xz full file
full_file_abs_path = os.path.join(patch_info.work_dir, to_marfile_entry.name)
shutil.copy2(to_marfile_entry.abs_path, full_file_abs_path)
bzip_file(full_file_abs_path)
xz_file(full_file_abs_path)
if os.path.getsize(patch_file_abs_path) < os.path.getsize(full_file_abs_path):
# Patch is smaller than file. Remove the file and add patch to manifest
@ -304,13 +304,17 @@ def process_explicit_remove_files(dir_path, patch_info):
list_file_path = os.path.join(dir_path, "Contents/Resources/removed-files")
if (os.path.exists(list_file_path)):
list_file = bz2.BZ2File(list_file_path,"r") # throws if doesn't exist
fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
os.close(fd)
exec_shell_cmd('xz -k -d --stdout "' + list_file_path + '" > "'+tmppath+'"')
list_file = open(tmppath)
lines = []
for line in list_file:
lines.append(line.strip())
list_file.close()
list_file.close()
os.remove(tmppath)
lines.sort(reverse=True)
for line in lines:
# Exclude any blank and comment lines.
@ -406,7 +410,7 @@ def create_partial_patch(from_dir_path, to_dir_path, patch_filename, shas, patch
patch_info.create_manifest_files()
# And construct the mar
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+string.join(patch_info.archive_files, ' ')
mar_cmd = 'mar -C '+patch_info.work_dir+' -c output.mar '+' '.join(patch_info.archive_files)
exec_shell_cmd(mar_cmd)
# Copy mar to final destination
@ -431,11 +435,18 @@ def get_buildid(work_dir):
print 'WARNING: application.ini not found, cannot find build ID'
return ''
file = bz2.BZ2File(ini)
fd, tmppath = tempfile.mkstemp('', 'tmp', os.getcwd())
os.close(fd)
exec_shell_cmd('xz -k -d --stdout "' + ini + '" > "'+tmppath+'"')
file = open(tmppath)
for line in file:
if line.find('BuildID') == 0:
file.close()
os.remove(tmppath)
return line.strip().split('=')[1]
print 'WARNING: cannot find build ID in application.ini'
print('WARNING: cannot find build ID in application.ini')
file.close()
os.remove(tmppath)
return ''
def decode_filename(filepath):
@ -486,7 +497,7 @@ def create_partial_patches(patches):
extract_mar(from_filename,work_dir_from)
from_decoded = decode_filename(from_filename)
from_buildid = get_buildid(work_dir_from)
from_shasum = sha.sha(open(from_filename).read()).hexdigest()
from_shasum = hashlib.sha1(open(from_filename, "rb").read()).hexdigest()
from_size = str(os.path.getsize(to_filename))
# Extract to mar into to dir
@ -495,14 +506,14 @@ def create_partial_patches(patches):
extract_mar(to_filename, work_dir_to)
to_decoded = decode_filename(from_filename)
to_buildid = get_buildid(work_dir_to)
to_shasum = sha.sha(open(to_filename).read()).hexdigest()
to_shasum = hashlib.sha1(open(to_filename, 'rb').read()).hexdigest()
to_size = str(os.path.getsize(to_filename))
mar_extract_time = time.time()
partial_filename = create_partial_patch(work_dir_from, work_dir_to, patch_filename, shas, PatchInfo(work_dir, ['update.manifest','updatev2.manifest','updatev3.manifest'],[]),forced_updates,['channel-prefs.js','update-settings.ini'])
partial_buildid = to_buildid
partial_shasum = sha.sha(open(partial_filename).read()).hexdigest()
partial_shasum = hashlib.sha1(open(partial_filename, "rb").read()).hexdigest()
partial_size = str(os.path.getsize(partial_filename))
metadata.append({