giuliomoro@229: # Copyright 2015,2016 Enzien Audio, Ltd. All Rights Reserved.
chris@160: 
chris@160: import argparse
giuliomoro@540: import datetime
chris@160: import getpass
chris@160: import json
chris@160: import os
giuliomoro@549: import requests # http://docs.python-requests.org/en/master/api/#exceptions
chris@160: import shutil
chris@160: import stat
giuliomoro@549: import sys
chris@160: import tempfile
chris@160: import time
chris@160: import urlparse
chris@160: import zipfile
chris@160: 
chris@160: class Colours:
chris@160:     purple = "\033[95m"
chris@160:     cyan = "\033[96m"
chris@160:     dark_cyan = "\033[36m"
chris@160:     blue = "\033[94m"
chris@160:     green = "\033[92m"
chris@160:     yellow = "\033[93m"
chris@160:     red = "\033[91m"
chris@160:     bold = "\033[1m"
chris@160:     underline = "\033[4m"
chris@160:     end = "\033[0m"
chris@160: 
giuliomoro@549: class ErrorCodes(object):
giuliomoro@549:     # NOTE(mhroth): this class could inherit from Enum, but we choose not to
giuliomoro@549:     # as to not require an additional dependency
giuliomoro@549:     # http://www.tldp.org/LDP/abs/html/exitcodes.html
giuliomoro@549:     # http://stackoverflow.com/questions/1101957/are-there-any-standard-exit-status-codes-in-linux
giuliomoro@549:     CODE_OK = 0 # success!
giuliomoro@549:     CODE_MAIN_NOT_FOUND = 3 # _main.pd not found
giuliomoro@549:     CODE_HEAVY_COMPILE_ERRORS = 4 # heavy returned compiler errors
giuliomoro@549:     CODE_UPLOAD_ASSET_TOO_LARGE = 5 # the size of the uploadable asset is too large
giuliomoro@549:     CODE_RELEASE_NOT_AVAILABLE = 6 # the requested release is not available
giuliomoro@549:     CODE_CONNECTION_ERROR = 7 # HTTPS connection could not be made to the server
giuliomoro@549:     CODE_CONNECTION_TIMEOUT = 8 # HTTPS connection has timed out
giuliomoro@549:     CODE_CONNECTION_400_500 = 9 # a 400 or 500 error has occured
giuliomoro@549:     CODE_EXCEPTION = 125 # a generic execption has occurred
giuliomoro@549: 
giuliomoro@549: class UploaderException(Exception):
giuliomoro@549:     def __init__(self, code, message=None, e=None):
giuliomoro@549:         self.code = code
giuliomoro@549:         self.message = message
giuliomoro@549:         self.e = e
giuliomoro@549: 
giuliomoro@403: # the maxmimum file upload size of 1MB
giuliomoro@549: __HV_MAX_UPLOAD_SIZE = 1 * 1024*1024
giuliomoro@403: 
chris@160: def __zip_dir(in_dir, zip_path, file_filter=None):
giuliomoro@403:     """Recursively zip an entire directory with an optional file filter
giuliomoro@403:     """
chris@160:     zf = zipfile.ZipFile(zip_path, mode="w", compression=zipfile.ZIP_DEFLATED)
chris@160:     for subdir, dirs, files in os.walk(in_dir):
giuliomoro@403:         for f in files:
giuliomoro@403:             if (file_filter is None) or (f.lower().split(".")[-1] in file_filter):
chris@160:                 zf.write(
giuliomoro@403:                     filename=os.path.join(subdir,f),
giuliomoro@403:                     arcname=os.path.relpath(os.path.join(subdir,f), start=in_dir))
chris@160:     return zip_path
chris@160: 
chris@160: def __unzip(zip_path, target_dir):
chris@160:     """Unzip a file to a given directory. All destination files are overwritten.
chris@160:     """
chris@160:     zipfile.ZipFile(zip_path).extractall(target_dir)
chris@160: 
chris@160: def main():
chris@160:     parser = argparse.ArgumentParser(
chris@160:         description="Compiles a Pure Data file.")
chris@160:     parser.add_argument(
chris@160:         "input_dir",
giuliomoro@229:         help="A directory containing _main.pd. All .pd files in the directory structure will be uploaded.")
chris@160:     parser.add_argument(
chris@160:         "-n", "--name",
chris@160:         default="heavy",
giuliomoro@229:         help="Patch name. If it doesn't exist on the Heavy site, the uploader will fail.")
chris@160:     parser.add_argument(
chris@160:         "-g", "--gen",
chris@160:         nargs="+",
chris@160:         default=["c"],
giuliomoro@229:         help="List of generator outputs. Currently supported generators are "
giuliomoro@229:             "'c', 'js', 'pdext', 'pdext-osx', 'unity', 'unity-osx', "
giuliomoro@229:             "'unity-win-x86', 'unity-win-x86_64', 'wwise', 'wwise-win-x86_64', "
giuliomoro@549:             "'vst2' ,'vst2-osx', 'vst2-win-x86_64', and 'vst2-win-x86'.")
chris@160:     parser.add_argument(
chris@160:         "-b",
giuliomoro@229:         help="All files will be placed in the output directory, placed in their own subdirectory corresponding to the generator name.",
giuliomoro@229:         action="count")
giuliomoro@229:     parser.add_argument(
giuliomoro@229:         "-y",
giuliomoro@229:         help="Extract only the generated C files. Static files are deleted. "
giuliomoro@229:             "Only effective for the 'c' generator.",
chris@160:         action="count")
chris@160:     parser.add_argument(
chris@160:         "-o", "--out",
chris@160:         nargs="+",
chris@160:         default=["./"], # by default
chris@160:         help="List of destination directories for retrieved files. Order should be the same as for --gen.")
chris@160:     parser.add_argument(
giuliomoro@403:         "-r", "--release",
giuliomoro@403:         help="Optionally request a specific release of Heavy to use while compiling.")
giuliomoro@403:     parser.add_argument(
chris@160:         "-d", "--domain",
chris@160:         default="https://enzienaudio.com",
chris@160:         help="Domain. Default is https://enzienaudio.com.")
chris@160:     parser.add_argument(
chris@160:         "-x",
chris@160:         help="Don't save the returned token.",
chris@160:         action="count")
chris@160:     parser.add_argument(
chris@160:         "-z",
chris@160:         help="Force the use of a password, regardless of saved token.",
chris@160:         action="count")
chris@160:     parser.add_argument(
chris@160:         "--noverify",
giuliomoro@229:         help="Don't verify the SSL connection. This is generally a very bad idea.",
chris@160:         action="count")
chris@160:     parser.add_argument(
chris@160:         "-v", "--verbose",
chris@160:         help="Show debugging information.",
chris@160:         action="count")
giuliomoro@229:     parser.add_argument(
giuliomoro@229:         "-t", "--token",
giuliomoro@549:         help="Use the specified token.")
chris@160:     args = parser.parse_args()
chris@160: 
giuliomoro@549:     try:
giuliomoro@549:         # set default values
giuliomoro@549:         domain = args.domain or "https://enzienaudio.com"
giuliomoro@549:         exit_code = ErrorCodes.CODE_OK
giuliomoro@549:         temp_dir = None
giuliomoro@549:         post_data = {}
chris@160: 
giuliomoro@549:         # token should be stored in ~/.heavy/token
giuliomoro@549:         token_path = os.path.expanduser(os.path.join("~/", ".heavy", "token"))
chris@160: 
giuliomoro@549:         if args.token is not None:
giuliomoro@549:             # check if token has been passed as a command line arg...
giuliomoro@549:             post_data["credentials"] = {"token": args.token}
giuliomoro@549:         elif os.path.exists(token_path) and not args.z:
giuliomoro@549:             # ...or if it is stored in the user's home directory
giuliomoro@549:             with open(token_path, "r") as f:
giuliomoro@549:                 post_data["credentials"] = {"token": f.read()}
giuliomoro@549:         else:
giuliomoro@549:             # otherwise, get the username and password
giuliomoro@549:             post_data["credentials"] = {
giuliomoro@549:                 "username": raw_input("Enter username: "),
giuliomoro@549:                 "password": getpass.getpass("Enter password: ")
giuliomoro@549:             }
giuliomoro@229: 
giuliomoro@549:         tick = time.time()
chris@160: 
giuliomoro@549:         # parse the optional release argument
giuliomoro@549:         if args.release:
giuliomoro@540:             # check the validity of the current release
giuliomoro@540:             releases_json = requests.get(urlparse.urljoin(domain, "/a/releases")).json()
giuliomoro@540:             if args.release in releases_json:
giuliomoro@540:                 today = datetime.datetime.now()
giuliomoro@540:                 valid_until = datetime.datetime.strptime(releases_json[args.release]["validUntil"], "%Y-%m-%d")
giuliomoro@540:                 if today > valid_until:
giuliomoro@540:                     print "{0}Warning:{1} The release \"{2}\" expired on {3}. It may be removed at any time!".format(
giuliomoro@540:                         Colours.yellow, Colours.end,
giuliomoro@540:                         args.release,
giuliomoro@540:                         releases_json[args.release]["validUntil"])
giuliomoro@540:                 elif (valid_until - today) <= datetime.timedelta(weeks=4):
giuliomoro@540:                     print "{0}Warning:{1} The release \"{2}\" will expire soon on {3}.".format(
giuliomoro@540:                         Colours.yellow, Colours.end,
giuliomoro@540:                         args.release,
giuliomoro@540:                         releases_json[args.release]["validUntil"])
giuliomoro@540:             else:
giuliomoro@540:                 print "{0}Error:{1} The release \"{2}\" is not available. Available releases are:".format(
giuliomoro@540:                     Colours.red, Colours.end,
giuliomoro@540:                     args.release)
giuliomoro@540:                 for k,v in releases_json.items():
giuliomoro@540:                     print "* {0} ({1})".format(
giuliomoro@540:                         k,
giuliomoro@540:                         v["releaseDate"])
giuliomoro@549:                 raise UploaderException(ErrorCodes.CODE_RELEASE_NOT_AVAILABLE)
giuliomoro@540: 
giuliomoro@549:             post_data["release"] = args.release
giuliomoro@403: 
giuliomoro@549:         # make a temporary directory
giuliomoro@549:         temp_dir = tempfile.mkdtemp(prefix="lroyal-")
chris@160: 
giuliomoro@549:         # zip up the pd directory into the temporary directory
chris@160:         if not os.path.exists(os.path.join(args.input_dir, "_main.pd")):
giuliomoro@549:             raise UploaderException(
giuliomoro@549:                 ErrorCodes.CODE_MAIN_NOT_FOUND,
giuliomoro@549:                 "Root Pd directory does not contain a file named _main.pd.")
chris@160:         zip_path = __zip_dir(
chris@160:             args.input_dir,
chris@160:             os.path.join(temp_dir, "archive.zip"),
chris@160:             file_filter={"pd"})
giuliomoro@403:         if os.stat(zip_path).st_size > __HV_MAX_UPLOAD_SIZE:
giuliomoro@549:             raise UploaderException(
giuliomoro@549:                 ErrorCodes.CODE_UPLOAD_ASSET_TOO_LARGE,
giuliomoro@549:                 "The target directory, zipped, is {0} bytes. The maximum upload size of 1MB.".format(
giuliomoro@549:                     os.stat(zip_path).st_size))
giuliomoro@549: 
giuliomoro@549:         post_data["name"] = args.name
giuliomoro@549: 
giuliomoro@549:         # the outputs to generate (always include c)
giuliomoro@549:         __SUPPORTED_GENERATOR_SET = {
giuliomoro@549:             "c", "js",
giuliomoro@549:             "pdext", "pdext-osx",
giuliomoro@549:             "unity", "unity-osx", "unity-win-x86", "unity-win-x86_64",
giuliomoro@549:             "wwise", "wwise-win-x86_64",
giuliomoro@549:             "vst2", "vst2-osx", "vst2-win-x86_64",
giuliomoro@549:         }
giuliomoro@549:         post_data["gen"] = list(({"c"} | {s.lower() for s in set(args.gen)}) & __SUPPORTED_GENERATOR_SET)
giuliomoro@549: 
giuliomoro@549:         # upload the job, get the response back
giuliomoro@549:         # NOTE(mhroth): multipart-encoded file can only be sent as a flat dictionary,
giuliomoro@549:         # but we want to send a json encoded deep dictionary. So we do a bit of a hack.
giuliomoro@549:         r = requests.post(
giuliomoro@549:             urlparse.urljoin(domain, "/a/heavy"),
giuliomoro@549:             data={"json":json.dumps(post_data)},
giuliomoro@549:             files={"file": (os.path.basename(zip_path), open(zip_path, "rb"), "application/zip")},
giuliomoro@549:             verify=False if args.noverify else True)
giuliomoro@549:         r.raise_for_status()
giuliomoro@549: 
giuliomoro@549:         """
giuliomoro@549:         {
giuliomoro@549:           "data": {
giuliomoro@549:             "compileTime": 0.05078411102294922,
giuliomoro@549:             "id": "mhroth/asdf/Edp2G",
giuliomoro@549:             "slug": "Edp2G",
giuliomoro@549:             "index": 3,
giuliomoro@549:             "links": {
giuliomoro@549:               "files": {
giuliomoro@549:                 "linkage": [
giuliomoro@549:                   {
giuliomoro@549:                     "id": "mhroth/asdf/Edp2G/c",
giuliomoro@549:                     "type": "file"
giuliomoro@549:                   }
giuliomoro@549:                 ],
giuliomoro@549:                 "self": "https://enzienaudio.com/h/mhroth/asdf/Edp2G/files"
giuliomoro@549:               },
giuliomoro@549:               "project": {
giuliomoro@549:                 "linkage": {
giuliomoro@549:                   "id": "mhroth/asdf",
giuliomoro@549:                   "type": "project"
giuliomoro@549:                 },
giuliomoro@549:                 "self": "https://enzienaudio.com/h/mhroth/asdf"
giuliomoro@549:               },
giuliomoro@549:               "self": "https://enzienaudio.com/h/mhroth/asdf/Edp2G",
giuliomoro@549:               "user": {
giuliomoro@549:                 "linkage": {
giuliomoro@549:                   "id": "mhroth",
giuliomoro@549:                   "type": "user"
giuliomoro@549:                 },
giuliomoro@549:                 "self": "https://enzienaudio.com/h/mhroth"
giuliomoro@549:               }
giuliomoro@549:             },
giuliomoro@549:             "type": "job"
giuliomoro@549:           },
giuliomoro@549:           "included": [
giuliomoro@549:             {
giuliomoro@549:               "filename": "file.c.zip",
giuliomoro@549:               "generator": "c",
giuliomoro@549:               "id": "mhroth/asdf/Edp2G/c",
giuliomoro@549:               "links": {
giuliomoro@549:                 "self": "https://enzienaudio.com/h/mhroth/asdf/Edp2G/c/file.c.zip"
giuliomoro@549:               },
giuliomoro@549:               "mime": "application/zip",
giuliomoro@549:               "type": "file"
giuliomoro@549:             }
giuliomoro@549:           ],
giuliomoro@549:           "warnings": [
giuliomoro@549:             {"details": "blah blah blah"}
giuliomoro@549:           ],
giuliomoro@549:           "meta": {
giuliomoro@549:             "token": "11AS0qPRmjTUHEMSovPEvzjodnzB1xaz"
giuliomoro@549:           }
giuliomoro@549:         }
giuliomoro@549:         """
giuliomoro@549:         # decode the JSON API response
giuliomoro@549:         reply_json = r.json()
giuliomoro@549:         if args.verbose:
giuliomoro@549:             print json.dumps(
giuliomoro@549:                 reply_json,
giuliomoro@549:                 sort_keys=True,
giuliomoro@549:                 indent=2,
giuliomoro@549:                 separators=(",", ": "))
giuliomoro@549: 
giuliomoro@549:         # update the api token, if present
giuliomoro@549:         if "token" in reply_json.get("meta",{}) and not args.x:
giuliomoro@549:             if args.token is not None:
giuliomoro@549:                 if reply_json["meta"]["token"] != args.token:
giuliomoro@549:                     print "WARNING: Token returned by API is not the same as the "
giuliomoro@549:                     "token supplied at the command line. (old = %s, new = %s)".format(
giuliomoro@549:                         args.token,
giuliomoro@549:                         reply_json["meta"]["token"])
giuliomoro@549:             else:
giuliomoro@549:                 if not os.path.exists(os.path.dirname(token_path)):
giuliomoro@549:                     # ensure that the .heavy directory exists
giuliomoro@549:                     os.makedirs(os.path.dirname(token_path))
giuliomoro@549:                 with open(token_path, "w") as f:
giuliomoro@549:                     f.write(reply_json["meta"]["token"])
giuliomoro@549:                 # force rw------- permissions on the file
giuliomoro@549:                 os.chmod(token_path, stat.S_IRUSR | stat.S_IWUSR)
giuliomoro@549: 
giuliomoro@549:         # print any warnings
giuliomoro@549:         for i,x in enumerate(reply_json.get("warnings",[])):
giuliomoro@549:             print "{3}) {0}Warning:{1} {2}".format(
giuliomoro@549:                 Colours.yellow, Colours.end, x["detail"], i+1)
giuliomoro@549: 
giuliomoro@549:         # check for errors
giuliomoro@549:         if len(reply_json.get("errors",[])) > 0:
giuliomoro@549:             for i,x in enumerate(reply_json["errors"]):
giuliomoro@549:                 print "{3}) {0}Error:{1} {2}".format(
giuliomoro@549:                     Colours.red, Colours.end, x["detail"], i+1)
giuliomoro@549:             raise UploaderException(ErrorCodes.CODE_HEAVY_COMPILE_ERRORS)
giuliomoro@549: 
giuliomoro@549:         # retrieve all requested files
giuliomoro@549:         for i,g in enumerate(args.gen):
giuliomoro@549:             file_url = __get_file_url_for_generator(reply_json, g)
giuliomoro@549:             if file_url and (len(args.out) > i or args.b):
giuliomoro@549:                 r = requests.get(
giuliomoro@549:                     file_url,
giuliomoro@549:                     cookies={"token": reply_json["meta"]["token"]},
giuliomoro@549:                     verify=False if args.noverify else True)
giuliomoro@549:                 r.raise_for_status()
giuliomoro@549: 
giuliomoro@549:                 # write the reply to a temporary file
giuliomoro@549:                 c_zip_path = os.path.join(temp_dir, "archive.{0}.zip".format(g))
giuliomoro@549:                 with open(c_zip_path, "wb") as f:
giuliomoro@549:                     f.write(r.content)
giuliomoro@549: 
giuliomoro@549:                 # unzip the files to where they belong
giuliomoro@549:                 if args.b:
giuliomoro@549:                     target_dir = os.path.join(os.path.abspath(os.path.expanduser(args.out[0])), g)
giuliomoro@549:                 else:
giuliomoro@549:                     target_dir = os.path.abspath(os.path.expanduser(args.out[i]))
giuliomoro@549:                 if not os.path.exists(target_dir):
giuliomoro@549:                     os.makedirs(target_dir) # ensure that the output directory exists
giuliomoro@549:                 __unzip(c_zip_path, target_dir)
giuliomoro@549: 
giuliomoro@549:                 if g == "c" and args.y:
giuliomoro@549:                     keep_files = ("_{0}.h".format(args.name), "_{0}.c".format(args.name))
giuliomoro@549:                     for f in os.listdir(target_dir):
giuliomoro@549:                         if not f.endswith(keep_files):
giuliomoro@549:                             os.remove(os.path.join(target_dir, f));
giuliomoro@549: 
giuliomoro@549:                 print "{0} files placed in {1}".format(g, target_dir)
giuliomoro@549:             else:
giuliomoro@549:                 print "{0}Warning:{1} {2} files could not be retrieved.".format(
giuliomoro@549:                     Colours.yellow, Colours.end,
giuliomoro@549:                     g)
giuliomoro@549: 
giuliomoro@549:             print "Job URL:", reply_json["data"]["links"]["self"]
giuliomoro@549:             print "Total request time: {0}ms".format(int(1000.0*(time.time()-tick)))
giuliomoro@549:             print "Heavy release:", reply_json.get("meta",{}).get("release", "default")
giuliomoro@549:     except UploaderException as e:
giuliomoro@549:         exit_code = e.code
giuliomoro@549:         if e.message:
giuliomoro@549:             print "{0}Error:{1} {2}".format(Colours.red, Colours.end, e.message)
giuliomoro@549:     except requests.ConnectionError as e:
giuliomoro@549:         print "{0}Error:{1} Could not connect to server. Is the server down? Is the internet down?\n{2}".format(Colours.red, Colours.end, e)
giuliomoro@549:         exit_code = ErrorCodes.CODE_CONNECTION_ERROR
giuliomoro@549:     except requests.ConnectTimeout as e:
giuliomoro@549:         print "{0}Error:{1} Connection to server timed out. The server might be overloaded. Try again later?\n{2}".format(Colours.red, Colours.end, e)
giuliomoro@549:         exit_code = ErrorCodes.CODE_CONNECTION_TIMEOUT
giuliomoro@549:     except requests.HTTPError as e:
giuliomoro@549:         print "{0}Error:{1} An HTTP error has occurred.\n{2}".format(Colours.red, Colours.end, e)
giuliomoro@549:         exit_code = ErrorCodes.CODE_CONNECTION_400_500
chris@160:     except Exception as e:
giuliomoro@549:         exit_code = ErrorCodes.CODE_EXCEPTION
giuliomoro@403:         print "{0}Error:{1} {2}".format(Colours.red, Colours.end, e)
giuliomoro@549:         print "Getting a weird error? Get the latest uploader at https://enzienaudio.com/static/uploader.py"
giuliomoro@549:     finally:
giuliomoro@549:         if temp_dir:
giuliomoro@549:             shutil.rmtree(temp_dir) # delete the temporary directory no matter what
chris@160: 
giuliomoro@549:     # exit and return the exit code
giuliomoro@549:     sys.exit(exit_code)
chris@160: 
chris@160: def __get_file_url_for_generator(json_api, g):
chris@160:     """Returns the file link for a specific generator.
chris@160:     Returns None if no link could be found.
chris@160:     """
chris@160:     for i in json_api["included"]:
chris@160:         if g == i["generator"]:
chris@160:             return i["links"]["self"]
chris@160:     return None # by default, return None
chris@160: 
chris@160: 
chris@160: 
chris@160: if __name__ == "__main__":
chris@160:     main()