marco@0: #!usr/bin/env/ python
marco@1: 
marco@12: """
marco@12: 
marco@15: SWORD2 DSpace bulk uploader - v0.6
marco@1: 
marco@1: A python script to submit large numbers of files to a SWORD2-compatible repository, specifically DSpace 1.8x.
marco@15: Built on the SWORD2 python client library: https://github.com/swordapp/python-client-sword2.
marco@1: 
marco@1: Dependencies:
marco@1: 
marco@1: - python 2.X
marco@1: 
marco@14: - sword2 library: https://github.com/swordapp/python-client-sword2 
marco@1: 
marco@1: -----------------------------------
marco@15: Updates log:
marco@15: 
marco@15: v0.6: 	- now uploading a directory will also maintain the path structure
marco@15: 		- introduced a file where to specify the server (server.cfg)
marco@15: v0.5: changed the default server to C4DM live server
marco@15: 
marco@15: -----------------------------------
marco@11:   Centre for Digital Music, Queen Mary, University of London
marco@11:   Copyright (c) 2012 Marco Fabiani
marco@11: 
marco@11:   Permission is hereby granted, free of charge, to any person
marco@11:   obtaining a copy of this software and associated documentation
marco@11:   files (the "Software"), to deal in the Software without
marco@11:   restriction, including without limitation the rights to use, copy,
marco@11:   modify, merge, publish, distribute, sublicense, and/or sell copies
marco@11:   of the Software, and to permit persons to whom the Software is
marco@11:   furnished to do so, subject to the following conditions:
marco@11: 
marco@11:   The above copyright notice and this permission notice shall be
marco@11:   included in all copies or substantial portions of the Software.
stephen@20: 
marco@11:   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
marco@11:   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
marco@11:   OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
marco@11:   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
marco@11:   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
marco@11:   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
marco@11:   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
marco@11:   OTHER DEALINGS IN THE SOFTWARE.
marco@1: -----------------------------------
marco@12: 
marco@11: A copy of this License can also be found in the COPYING file distributed with the source code.
marco@1: """
marco@0: 
stephen@20: import argparse, getpass, zipfile, os, sword2.http_layer
marco@0: from sword2 import *
marco@0: 
marco@0: # Parse arguments
stephen@20: parser = argparse.ArgumentParser(description="Bulk upload to DSpace using SWORD v2.",epilog="If the submission is created successfully, it will remain open to be completed with the necessary metadata and licenses, using the DSpace web interface. The submission can be found in the \"My Account -> Submissions\" section of the user's area.")
marco@0: parser.add_argument("data", type=str, nargs=1,
stephen@20: 				   help="Accepts: METSDSpaceSIP and BagIt packages, simple zip files, directories, single files. NOTE: METSDSpaceSIP packages are only accepted by Collections with a workflow!")
stephen@20: parser.add_argument("--username", dest="user_name", type=str, nargs=1, help="DSpace username.")
stephen@20: parser.add_argument("--password", dest="password", type=str, nargs=1, help="DSpace password.")
stephen@21: parser.add_argument("--timeout", dest="timeout", type=float, nargs=1, default=[30.0], help="Timeout for response for connections. Make sure this is long enough to allow files to be uploaded.")
marco@0: parser.add_argument("--title", dest="title", type=str,nargs=1, help="Title (ignored for METS packages).")
stephen@20: parser.add_argument("--author", dest="author", type=str, nargs="+", help="Author(s) (ignored for METS packages). Accepts multiple entries in the format \"Surname, Name\"")
stephen@20: parser.add_argument("--date", dest="date", type=str, nargs=1, help="Date of creation (string) (ignored for METS packages).")
stephen@20: parser.add_argument("--zip", action="store_true", dest="zip", default=False, help="If \"data\" is a directory, compress it and post it as a single file. The zip file will be saved along with the individual files.")
stephen@20: parser.add_argument("--servicedoc", dest="sd", type=str, nargs=1, help="Url of the SWORD v2 service document (default: use server.cfg if available, otherwise http://c4dm.eecs.qmul.ac.uk/rdr/swordv2/servicedocument")
marco@0: 
marco@0: args = parser.parse_args()
marco@0: data = args.data[0]
stephen@20: timeout = args.timeout[0]
marco@13: if args.zip:
stephen@20: 	storeZip = True
marco@13: else:
stephen@20: 	storeZip = False
marco@8: 
stephen@20: if args.sd is None:
marco@15: 	try:
marco@15: 		f = open("server.cfg", "r")
marco@15: 		sd = f.readline()
marco@15: 		print "server.cfg: ", sd
marco@15: 	except:
marco@15: 		sd = "http://c4dm.eecs.qmul.ac.uk/rdr/swordv2/servicedocument"
marco@0: else:
marco@13: 	sd = args.sd[0]
marco@0: 
stephen@20: class swordConnection(object):
stephen@20: 	def __init__(self):
stephen@20: 		self.serverConnection = None
stephen@20: 		self.connected = False
stephen@20: 		self.name = ""
marco@0: 
stephen@20: 	def connect(self, timeout=30.0):
stephen@20: 		self.serverConnection = None
stephen@20: 		self.connected = False
stephen@20: 		httpImp = sword2.http_layer.HttpLib2Layer(".cache", timeout=timeout)
stephen@20: 		print "Connection timeout is ", timeout, "seconds."
stephen@20: 		# Connect to SWORD server: it will always try to authenticate (no anonymous submissions!
stephen@20: 		attempts = 3 #  Number of attempts left to connect to server
stephen@20: 		while attempts>0 and not self.connected:
stephen@20: 			print "Connecting to SWORD server. Remaining attempts: ", attempts
stephen@20: 			# Try to login, get service document
stephen@20: 			# Get username and password
stephen@20: 			if args.user_name is None:
stephen@20: 				user_name = raw_input("Username: ")
stephen@20: 			else:
stephen@20: 				user_name = args.user_name[0]
stephen@20: 				print "Username: ",user_name
stephen@20: 
stephen@20: 			if args.password is None:
stephen@20: 				user_pass = getpass.getpass("Password:")
stephen@20: 			else:
stephen@20: 				user_pass = args.password[0]
stephen@20: 			# Connect to the server
stephen@20: 
stephen@20: 			self.serverConnection = Connection(sd, user_name=user_name, user_pass=user_pass,keep_history=False,http_impl=httpImp)
stephen@20: 
stephen@20: 			# Get service document
stephen@20: 			try:
stephen@20: 				self.serverConnection.get_service_document()
stephen@20: 			except: # Server error
stephen@20: 				print "Server unreachable!"
stephen@20: 				break
stephen@20: 
stephen@20: 			if self.serverConnection.sd is not None:
stephen@20: 				self.connected = True
stephen@20: 			else:
stephen@20: 				attempts-=1
stephen@20: 				print "Incorrect username and/or password"
stephen@20: 
stephen@20: 		if not self.connected:
stephen@20: 			# Failed to connect to SWORD v2 Server
stephen@20: 			print "Couldn't connect to the server."
stephen@20: 			if attempts == 0:
stephen@20: 				raise Exception, "Invalid credentials entered 3 times."
stephen@20: 			else:
stephen@20: 				raise Exception, "Unable to connect to server"
marco@0: 		else:
stephen@20: 			self.name = self.serverConnection.workspaces[0][0]
stephen@20: 
stephen@20: 	def selectCollection(self):
marco@0: 		# List available collections
marco@0: 		print "Available Collections: "
stephen@20: 		numColl = len(self.serverConnection.workspaces[0][1])
marco@0: 		for ctr in range(numColl):
stephen@20: 			coll = self.serverConnection.workspaces[0][1][ctr]
marco@0: 			print ctr+1,":",coll.title
marco@0: 		# Select a collection to deposit into
marco@14: 		sel = "0"
stephen@20: 		while (not sel.isdigit()) or int(sel)<=0 or int(sel)>numColl:
marco@14: 			sel = raw_input("Select a Collection to submit your files into: ")
marco@14: 		sel = int(sel)
stephen@20: 		collection = swordCollection(self, self.serverConnection.workspaces[0][1][sel-1])
stephen@20: 		return collection
stephen@20: 
stephen@20: 
stephen@20: class swordCollection(object):
stephen@20: 	def __init__(self, connection, collection):
stephen@20: 		self.connection = connection
stephen@20: 		self.serverCollection = collection
stephen@20: 
stephen@20: 	def title(self):
stephen@20: 		return self.serverCollection.title
stephen@20: 
stephen@20: 	def createItem(self, metadata_entry, in_progress=True):
stephen@20: 		creationReceipt = self.connection.serverConnection.create(col_iri = self.serverCollection.href, metadata_entry = metadata_entry, in_progress=in_progress)
stephen@20: 		return swordItem(self.connection, self, creationReceipt)
stephen@20: 
stephen@20: 	def createItemFromFile(self, file, metadata_entry, in_progress=True):
stephen@20: 		depositReceipt = None
stephen@20: 		payload = open(file.path, "rb")
stephen@20: 		try:
stephen@20: 			deposit_receipt = self.connection.serverConnection.create(col_iri = self.serverCollection.href,
stephen@20: 				payload = payload,
stephen@20: 				filename = file.filename,
stephen@20: 				mimetype = file.mimetype,
stephen@20: 				packaging = file.packaging,
stephen@20: 				in_progress = in_progress)
stephen@20: 			print type, " submission successful."
stephen@20: 		except:
stephen@20: 			print "Error! Couldn't submit the file!"
stephen@20: 			if type == "METS": # Just guessing: not sure this is the problem...
stephen@20: 				print "To submit a METS package, the collection MUST have a workflow!"
stephen@20: 		payload.close()
stephen@20: 
stephen@20: 		return swordItem(self.connection, self, depositReceipt)
stephen@20: 
stephen@20: class swordItem(object):
stephen@20: 	def __init__(self, connection, collection, receipt):
stephen@20: 		self.connection = connection
stephen@20: 		self.serverCollection = collection
stephen@20: 		self.receipt = receipt
stephen@20: 
stephen@20: 	def addFile(self, file):
stephen@20: #		print "Adding to", self.receipt.edit_media
stephen@20: #		print str(file)
stephen@20: 		payload = open(file.path, "rb")
stephen@20: 		print "Uploading file ", file.filename,
stephen@20: 		file.deposit_receipt = self.connection.serverConnection.add_file_to_resource(self.receipt.edit_media,
stephen@20: 			payload = payload,
stephen@20: 			filename = file.filename,
stephen@20: 			mimetype = file.mimetype,
stephen@20: 			packaging = file.packaging)
stephen@20: 		payload.close()
stephen@20: 		print "[uploaded]"
stephen@20: 
stephen@20: 	def updateMetadata(self, metadataEntry, in_progress=True):
stephen@20: 		try:
stephen@20: 			update_receipt = self.connection.serverConnection.update(dr = self.receipt, metadata_entry = metadataEntry, in_progress = in_progress)
stephen@20: 			print "Metadata update successful."
stephen@20: 		except:
stephen@20: 			print "Server error"
stephen@20: 			raise
stephen@20: 
stephen@20: # Class to encapsulate a SWORD2 payload file
stephen@20: class swordFile(object):
stephen@20: 	def __init__(self, path, filename=None):
stephen@20: 		self.path = path
stephen@20: 		self.deposit_receipt = None
stephen@20: 		if filename is None:
stephen@20: 			self.filename = os.path.basename(path)
marco@0: 		else:
stephen@20: 			self.filename = filename
stephen@20: 		# Default to a basic binary file
stephen@20: 		self.mimetype = "application/octet+stream"
stephen@20: 		self.packaging = 'http://purl.org/net/sword/package/Binary'
stephen@20: 
stephen@20: 	def __str__(self):
stephen@20: 		return "path:" + str(self.path) + ", filename:" + str(self.filename) + ", mimetype:" + str(self.mimetype) + ", packaging:" + str(self.packaging)
stephen@20: 
stephen@20: def getSubmissionData(args, data):
stephen@20: 	# Create a submission
stephen@20: 	filesList = []
stephen@20: 	temp = False # Delete temp files
stephen@20: 	packaging = None
stephen@20: 	# If folder
stephen@20: 	if os.path.isdir(data):
stephen@20: 		if args.zip: # If zip option, zip all the files and maintain the structure, but start from the base only...
stephen@20: 			dataName = os.path.basename(os.path.normpath(data))
stephen@20: 			if args.title is not None:
stephen@20: 				zipFile = args.title[0].replace(" ","_")+".zip"
stephen@20: 			else:
stephen@20: 				zipFile = dataName.replace(" ","_")+".zip"
stephen@20: 			myZip = zipfile.ZipFile(zipFile, "w")
stephen@20: 			# get the directory structure
stephen@20: 			print "Creating a zip archive for submission..."
stephen@20: 			for root, dirs, files in os.walk(data):
stephen@20: 				for name in files:
stephen@20: 					if not name.startswith('.'): # Do not upload hidden files, OSX/linux
stephen@20: 						# Remove spaces and square brackets
stephen@20: 						myZip.write(os.path.join(root,name),
stephen@20: 							os.path.relpath(os.path.join(root,name),data).replace(" ","_").replace("[","(").replace("]",")"))
stephen@20: 			filesList.append(zipFile)
stephen@20: 			myZip.close()
stephen@20: 			packaging = "http://purl.org/net/sword/package/SimpleZip"
stephen@20: 			type = "SimpleZip"
stephen@20: 			temp = True
marco@4: 		else:
stephen@20: 			# Create a list of files to upload
stephen@20: 			for root, dirs, files in os.walk(data):
stephen@20: 				for name in files:
stephen@20: 					if not name.startswith('.'):
stephen@20: 						filesList.append(os.path.join(root,name))
stephen@20: 			type = "multiple files"
stephen@20: 	elif zipfile.is_zipfile(data):
stephen@20: 		# This is a zip file
stephen@20: 		filesList.append(data)
stephen@20: 		myZip = zipfile.ZipFile(data)
stephen@20: 		if "mets.xml" in myZip.namelist():
stephen@20: 			# This is a METS package
stephen@20: 			packaging = "http://purl.org/net/sword/package/METSDSpaceSIP"
stephen@20: 			type = "METS"
stephen@20: 			in_progress = False
stephen@20: 		elif "bagit.txt" in "".join(myZip.namelist()):
stephen@20: 			# This is a BagIt package
stephen@20: 			packaging = "http://purl.org/net/sword/package/BagIt"
stephen@20: 			type = "BAGIT"
stephen@20: 		else:
stephen@20: 			# This is a simple zip file
stephen@20: 			packaging = "http://purl.org/net/sword/package/SimpleZip"
stephen@20: 			type = "SimpleZip"
stephen@20: 		myZip.close()
stephen@20: 	elif os.path.isfile(data): # This is a single file
stephen@20: 		filesList.append(data)
stephen@20: 		type = "single file"
stephen@20: 	else:
stephen@20: 		raise Exception, "Couldn't find the data."
stephen@20: 
stephen@20: 	submissionData = {"files": filesList, "packaging": packaging, "type":type, "isTemporaryFile":temp}
stephen@20: 	return submissionData
stephen@20: 
stephen@20: def setupMetadataEntry(args):
stephen@20: 	# Create a metadata entry
stephen@20: 	if (args.title is not None) or (args.author is not None) or (args.date is not None):
stephen@20: 		entry = Entry()
stephen@20: 		if  args.title is not None:
stephen@20: 			entry.add_fields(dcterms_title = args.title[0])
stephen@20: 		if args.author is not None:
stephen@20: 			for creator in args.author:
stephen@20: 				entry.add_fields(dcterms_creator=creator)
stephen@20: 		if args.date is not None:
stephen@20: 			entry.add_fields(dcterms_created = args.date[0])
stephen@20: 	else:
stephen@20: 		entry = None
stephen@20: 	return entry
stephen@20: 
stephen@20: try:
stephen@20: 	serverConnection = swordConnection()
stephen@20: 	serverConnection.connect(timeout)
stephen@20: 	print "------------------------"
stephen@20: 	print "Welcome to the", serverConnection.name, "repository"
stephen@20: 
stephen@20: 	collectionForItem = serverConnection.selectCollection()
stephen@20: 	print "Selected Collection:", collectionForItem.title()
stephen@20: 
stephen@20: 	submissionData = getSubmissionData(args, data)
stephen@20: 
stephen@20: 	print "------------------------"
stephen@20: 	print "This is a", submissionData["type"], "submission"
stephen@20: 
stephen@20: 	metadataEntry = setupMetadataEntry(args)
stephen@20: 
stephen@20: 	# Select what to do
stephen@20: 	if (submissionData["type"] == "single file") or (submissionData["type"] == "multiple files"): # Use the single file upload procedure
stephen@20: 		try:
stephen@20: 			# Create the metadata entry with ATOM
stephen@20: 			print "------------------------"
stephen@20: 			print "Creating the", submissionData["type"], "item... "
stephen@20: 			if metadataEntry is None:
stephen@20: 				metadataEntry = Entry(dcterms_title=(os.path.basename(data)))
stephen@20: 			collectionItem = collectionForItem.createItem(metadata_entry = metadataEntry, in_progress=True)
stephen@20: 			print "Item created"
stephen@20: 
stephen@20: 			# Create a list of files to upload
stephen@20: 			if submissionData["type"] == "single file":
stephen@20: 				payLoadList = [swordFile(submissionData["files"][0])]
stephen@20: 			else:
marco@15: 				# Get the longest common path in order to send the correct filename to keep the structure
stephen@20: 				common = os.path.commonprefix(submissionData["files"])
stephen@20: 				payLoadList=[]
stephen@20: 				for f in submissionData["files"]:
stephen@20: 					filename = os.path.relpath(f, common)
stephen@20: 					payLoadList.append(swordFile(f, filename))
stephen@20: 
stephen@20: 			# Upload the files
stephen@20: 			for payload in payLoadList:
stephen@20: 				collectionItem.addFile(payload)
stephen@20: 		except HTTPResponseError:
stephen@20: 			print "Bad request"
stephen@20: 	else:
stephen@20: 		# Send the zip file and let the ingester do its job
stephen@20: 		if (type == "SimpleZip") or (type=="BAGIT"):
stephen@20: 			in_progress = True
stephen@20: 			# FIXME: we don't want to write silly things in dc.description!
marco@4: 		else:
stephen@20: 			in_progress = False
stephen@20: 
stephen@20: 		payload = swordFile(submissionData["files"][0])
stephen@20: 		payload.mimetype = "application/zip"
stephen@20: 		payload.packaging = submissionData["packaging"]
stephen@20: 		item = collectionForItem.createItemFromFile(payload, in_progress)
stephen@20: 
stephen@20: 		# If some of the additional arguments for author, title, date etc. have been specified, update the metadata (only SimpleZip)
stephen@20: 		if type == "SimpleZip":
stephen@20: 			if metadataEntry is None:
stephen@20: 				metadataEntry = Entry(dcterms_title=(os.path.basename(submissionData["files"][0])))
stephen@20: 
stephen@20: 			# in_progress is True: we don't want to close the submission
stephen@20: 			item.updateMetadata(metadataEntry, in_progress=True)
stephen@20: 
stephen@20: 			# If we want to store the zip file along with the individual files (Only SimpleZip)
stephen@20: 			if storeZip:
marco@4: 				try:
stephen@20: 					zipPayload = swordFile(submissionData["files"][0], os.path.basename(submissionData["files"][0]).replace(" ", "_"))
stephen@20: 					zipPayload.mimetype = "application/zip"
stephen@20: 					zipPayload.packaging = 'http://purl.org/net/sword/package/Binary'
stephen@20: 					item.addFile(zipPayload)
stephen@20: 					print "Zip file successfully added to the bitstreams."
marco@4: 				except:
stephen@20: 					print "Server error: could not add the zip file to the resources"
marco@0: 
stephen@20: 		if submissionData["isTemporaryFile"]:
stephen@20: 			os.remove(submissionData["files"][0])
marco@4: 
stephen@20: 	print "------------------------"
stephen@20: 	print "You will find the submission in the \"Submissions\" list in your DSpace account. To complete/edit it with metadata and licenses, click on the title and then on \"Resume\"."
stephen@20: 
marco@0: except KeyboardInterrupt:
marco@0: 	print "------------------------"
marco@3: 	print "\nSubmission aborted by user."