Add in '--move' flag for deletion on success. Close #390

.gitignore

@@ -46,6 +46,10 @@ nosetests.xml
 #.idea/
 #.idea/workspace.xml
 #*.rst
+#
+**/.DS_Store
 bypy/test/**/*.bin
 bypy/test/downdir/
 bypy/test/sharedir/
+.vscode/tags
+

.vscode/.ropeproject/config.py

+# The default ``config.py``
+# flake8: noqa
+
+
+def set_prefs(prefs):
+    """This function is called before opening the project"""
+
+    # Specify which files and folders to ignore in the project.
+    # Changes to ignored resources are not added to the history and
+    # VCSs.  Also they are not returned in `Project.get_files()`.
+    # Note that ``?`` and ``*`` match all characters but slashes.
+    # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc'
+    # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc'
+    # '.svn': matches 'pkg/.svn' and all of its children
+    # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o'
+    # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o'
+    prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject',
+                                  '.hg', '.svn', '_svn', '.git', '.tox']
+
+    # Specifies which files should be considered python files.  It is
+    # useful when you have scripts inside your project.  Only files
+    # ending with ``.py`` are considered to be python files by
+    # default.
+    #prefs['python_files'] = ['*.py']
+
+    # Custom source folders:  By default rope searches the project
+    # for finding source folders (folders that should be searched
+    # for finding modules).  You can add paths to that list.  Note
+    # that rope guesses project source folders correctly most of the
+    # time; use this if you have any problems.
+    # The folders should be relative to project root and use '/' for
+    # separating folders regardless of the platform rope is running on.
+    # 'src/my_source_folder' for instance.
+    #prefs.add('source_folders', 'src')
+
+    # You can extend python path for looking up modules
+    #prefs.add('python_path', '~/python/')
+
+    # Should rope save object information or not.
+    prefs['save_objectdb'] = True
+    prefs['compress_objectdb'] = False
+
+    # If `True`, rope analyzes each module when it is being saved.
+    prefs['automatic_soa'] = True
+    # The depth of calls to follow in static object analysis
+    prefs['soa_followed_calls'] = 0
+
+    # If `False` when running modules or unit tests "dynamic object
+    # analysis" is turned off.  This makes them much faster.
+    prefs['perform_doa'] = True
+
+    # Rope can check the validity of its object DB when running.
+    prefs['validate_objectdb'] = True
+
+    # How many undos to hold?
+    prefs['max_history_items'] = 32
+
+    # Shows whether to save history across sessions.
+    prefs['save_history'] = True
+    prefs['compress_history'] = False
+
+    # Set the number spaces used for indenting.  According to
+    # :PEP:`8`, it is best to use 4 spaces.  Since most of rope's
+    # unit-tests use 4 spaces it is more reliable, too.
+    prefs['indent_size'] = 4
+
+    # Builtin and c-extension modules that are allowed to be imported
+    # and inspected by rope.
+    prefs['extension_modules'] = []
+
+    # Add all standard c-extensions to extension_modules list.
+    prefs['import_dynload_stdmods'] = True
+
+    # If `True` modules with syntax errors are considered to be empty.
+    # The default value is `False`; When `False` syntax errors raise
+    # `rope.base.exceptions.ModuleSyntaxError` exception.
+    prefs['ignore_syntax_errors'] = False
+
+    # If `True`, rope ignores unresolvable imports.  Otherwise, they
+    # appear in the importing namespace.
+    prefs['ignore_bad_imports'] = False
+
+    # If `True`, rope will insert new module imports as
+    # `from <package> import <module>` by default.
+    prefs['prefer_module_from_imports'] = False
+
+    # If `True`, rope will transform a comma list of imports into
+    # multiple separate import statements when organizing
+    # imports.
+    prefs['split_imports'] = False
+
+    # If `True`, rope will sort imports alphabetically by module name
+    # instead of alphabetically by import statement, with from imports
+    # after normal imports.
+    prefs['sort_imports_alphabetically'] = False
+
+
+def project_opened(project):
+    """This function is called after opening the project"""
+    # Do whatever you like here!

HISTORY.md

 ### Version History:
 
+- 1.5.8: Add `--move` flag to delete source files/directories on successfull transfers
 - 1.5.7: Reduce multiprocess timeout to 49 days, to accommodate Python 3 on Windows
 - 1.5.6: Downloading using downloader also retries
 - 1.5.5: Minor: Improve 'multiprocess' installation prompts

HISTORY.rst

 Version History:
 ~~~~~~~~~~~~~~~~
 
+-  1.5.8: Add ``--move`` flag to delete source files/directories on
+   successfull transfers
 -  1.5.7: Reduce multiprocess timeout to 49 days, to accommodate Python
    3 on Windows
 -  1.5.6: Downloading using downloader also retries

bypy/bypy.py

@@ -301,6 +301,7 @@ class ByPy(object):
 		mirror = '',
 		selectmirror = False,
 		resumedl_revertcount = const.DefaultResumeDlRevertCount,
+		deletesource = False,
 		verbose = 0, debug = False,
 		configdir = const.ConfigDir,
 		requester = RequestsRequester,
@@ -370,6 +371,10 @@ class ByPy(object):
 		self.__followlink = followlink
 		self.__rapiduploadonly = rapiduploadonly
 		self.__resumedl_revertcount = resumedl_revertcount
+		self.__deletesource = deletesource
+		if deletesource:
+			self.pd("Forcing verification since we will delete source for successful transfers.")
+			self.__verify = True
 		self.processes = processes
 
 		# these two variables are without leadning double underscaore "__" as to export the as public,
@@ -1111,6 +1116,26 @@ Possible fixes:
 		''' Usage: refreshtoken - refresh the access token '''
 		return self.__refresh_token()
 
+	def __remove_remote_on_success(self, remotepath):
+		if self.__deletesource:
+			self.pd("Removing remote path '{}' after successful download.".format(remotepath))
+			result = self.__delete(remotepath)
+			if result == const.ENoError:
+				self.pd("Remote path '{}' removed.".format(remotepath))
+			else:
+				perr("Failed to remove remote path '{}'.".format(remotepath))
+			return result
+
+	def __remove_local_on_success(self, localpath):
+		if self.__deletesource:
+			self.pd("Removing local path '{}' after successful upload.".format(localpath))
+			result = cachedm.remove_path_and_cache(localpath)
+			if result == const.ENoError:
+				self.pd("Local path '{}' removed.".format(localpath))
+			else:
+				perr("Failed to remove local path '{}'.".format(localpath))
+			return result
+
 	def info(self):
 		return self.quota()
 
@@ -1635,16 +1660,21 @@ get information of the given path (dir / file) at Baidu Yun.
 
 	def __upload_dir(self, localpath, remotepath, ondup = 'overwrite'):
 		self.pd("Uploading directory '{}' to '{}'".format(localpath, remotepath))
+		result = const.ENoError
 		if Pool and self.processes > 1:
-			return self.__upload_dir_multi(localpath, remotepath, ondup)
+			result = self.__upload_dir_multi(localpath, remotepath, ondup)
 		else:
-			return self.__upload_dir_single(localpath, remotepath, ondup)
+			result =  self.__upload_dir_single(localpath, remotepath, ondup)
+		if result == const.ENoError:
+			self.__remove_local_on_success(localpath)
+		return result
 
 	def __upload_file(self, localpath, remotepath, ondup = 'overwrite'):
 		# TODO: this is a quick patch
 		if not self.__shallinclude(localpath, remotepath, True):
 			# since we are not going to upload it, there is no error
-			return const.ENoError
+			#return const.ENoError
+			return const.ESkipped
 
 		self.__current_file = localpath
 		self.__current_file_size = getfilesize(localpath)
@@ -1677,12 +1707,18 @@ get information of the given path (dir / file) at Baidu Yun.
 							self.__current_file_size))
 				else:
 					self.pv("'{}' can't be rapidly uploaded, so it's skipped since we are in the rapid-upload-only mode.".format(localpath))
-
-			return result
-		elif not self.__rapiduploadonly:
+					result = const.ESkipped
+		elif self.__rapiduploadonly:
+			self.pv("'{}' is too small to be rapidly uploaded, so it's skipped since we are in the rapid-upload-only mode.".format(localpath))
+			result = const.ESkipped
+		else:
 			# very small file, must be uploaded manually and no slicing is needed
 			self.pd("'{}' is small and being non-slicing uploaded.".format(self.__current_file))
-			return self.__upload_one_file(localpath, remotepath, ondup)
+			result = self.__upload_one_file(localpath, remotepath, ondup)
+
+		if result == const.ENoError:
+			self.__remove_local_on_success(localpath)
+		return result
 
 	def upload(self, localpath = '', remotepath = '', ondup = "overwrite"):
 		''' Usage: upload [localpath] [remotepath] [ondup] - \
@@ -1945,12 +1981,12 @@ try to create a file at PCS by combining slices, having MD5s specified
 
 		return result
 
-
 	def __downfile(self, remotefile, localfile):
 		# TODO: this is a quick patch
 		if not self.__shallinclude(localfile, remotefile, False):
 			# since we are not going to download it, there is no error
-			return const.ENoError
+			#return const.ENoError
+			return const.ESkipped
 
 		result = const.ENoError
 		rfile = remotefile
@@ -1972,12 +2008,14 @@ try to create a file at PCS by combining slices, having MD5s specified
 			if const.ENoError == self.__verify_current_file(self.__remote_json, False) \
 				and not (self.__downloader[:5] == const.DownloaderAria2 and os.path.exists(localfile + '.aria2')):
 				self.pd("Same local file '{}' already exists, skip downloading".format(localfile))
+				self.__remove_remote_on_success(remotefile)
 				return const.ENoError
 			else:
 				if not self.shalloverwrite("Same-name locale file '{}' exists but is different, "
 						"do you want to overwrite it? [y/N]".format(localfile)):
 					pinfo("Same-name local file '{}' exists but is different, skip downloading".format(localfile))
-					return const.ENoError
+					#return const.ENoError
+					return const.ESkipped
 
 			if self.__resumedownload and \
 				self.__compare_size(self.__current_file_size, self.__remote_json) == 2:
@@ -1993,7 +2031,8 @@ try to create a file at PCS by combining slices, having MD5s specified
 			if not self.shalloverwrite("Same-name directory '{}' exists, "
 				"do you want to remove it? [y/N]".format(localfile)):
 				pinfo("Same-name directory '{}' exists, skip downloading".format(localfile))
-				return const.ENoError
+				#return const.ENoError
+				return const.ESkipped
 
 			self.pv("Directory with the same name '{}' exists, removing ...".format(localfile))
 			result = removedir(localfile, self.verbose)
@@ -2011,9 +2050,14 @@ try to create a file at PCS by combining slices, having MD5s specified
 				return result
 
 		if self.__downloader[:5] == const.DownloaderAria2:
-			return self.__down_aria2c(rfile, localfile)
+			result = self.__down_aria2c(rfile, localfile)
 		else:
-			return self.__downchunks(rfile, offset)
+			result = self.__downchunks(rfile, offset)
+
+		if result == const.ENoError:
+			self.__remove_remote_on_success(remotefile)
+
+		return result
 
 	def downfile(self, remotefile, localpath = ''):
 		''' Usage: downfile <remotefile> [localpath] - \
@@ -2185,10 +2229,14 @@ To stream a file, you can use the 'mkfifo' trick with omxplayer etc.:
 			"Directory Download")
 
 	def __downdir(self, rpath, lpath):
+		result = const.ENoError
 		if Pool and self.processes > 1:
-			return self.__downdir_multi(rpath, lpath)
+			result = self.__downdir_multi(rpath, lpath)
 		else:
-			return self.__downdir_single(rpath, lpath)
+			result = self.__downdir_single(rpath, lpath)
+		if result == const.ENoError:
+			self.__remove_remote_on_success(rpath)
+		return result
 
 	def downdir(self, remotepath = None, localpath = None):
 		''' Usage: downdir [remotedir] [localdir] - \
@@ -2240,7 +2288,8 @@ download a remote directory (recursively) / file
 		# the code still works because Baidu Yun doesn't require
 		# parent directory to exist remotely to upload / create a file
 		if not self.__shallinclude('.', rpath, True):
-			return const.ENoError
+			#return const.ENoError
+			return const.ESkipped
 
 		self.pd("Making remote directory '{}'".format(rpath))
 
@@ -2711,6 +2760,8 @@ if not specified, it defaults to the root directory
 			if subresult != const.ENoError:
 				result = subresult
 
+		if result == const.ENoError:
+			self.__remove_remote_on_success(rpath)
 		return result
 
 	def __syncup_diff_one(self, rpath, localdir, d):
@@ -2832,6 +2883,8 @@ if not specified, it defaults to the root directory
 			subresult = self.__syncup_delete_remote(rpath, remote)
 			if subresult != const.ENoError:
 				result = subresult
+		if result == const.ENoError:
+			self.__remove_local_on_success(localdir)
 		return result
 
 	def dumpcache(self):
@@ -3422,6 +3475,9 @@ def getparser():
 		dest="resumedl_revertcount", default=const.DefaultResumeDlRevertCount,
 		type=int, metavar='RCOUNT',
 		help="Revert back at least %(metavar)s download chunk(s) and align to chunk boundary when resuming the download. A negative value means NO reverts. [default: %(default)s]")
+	parser.add_argument("--move",
+		dest="deletesource", action="store_true",
+		help="Delete source files/directories after download/upload/syncdown/syncup is successful (This will force verification of the files). [default: %(default)s]")
 	if Pool:
 		parser.add_argument(const.MultiprocessOption,
 			dest="processes", default=const.DefaultProcessCount, type=int,
@@ -3580,6 +3636,7 @@ def main(argv=None): # IGNORE:C0111
 				'selectmirror': args.selectmirror,
 				'configdir': args.configdir,
 				'resumedl_revertcount': args.resumedl_revertcount,
+				'deletesource': args.deletesource,
 				'downloader': args.downloader,
 				'downloader_args': dl_args,
 				'verbose': args.verbose,

bypy/cached.py

@@ -20,7 +20,8 @@ from .util import (
 	pdbg, pinfo, perr,
 	getfilesize, getfilemtime_int,
 	joinpath, formatex,
-	jsonload, jsondump)
+	jsonload, jsondump,
+	removepath)
 
 pr = util.pr
 
@@ -257,6 +258,32 @@ class cached(object):
 						cached.cache[absdir] = files
 		cached.savecache()
 
+	@staticmethod
+	def remove(path):
+		def notfound():
+			pdbg("Failed to delete cache: Path '{}' not found in cache.".format(path))
+
+		dir, file = os.path.split(path)
+		absdir = os.path.abspath(dir)
+		if absdir in cached.cache:
+			entry = cached.cache[absdir]
+			if file in entry:
+				del entry[file]
+				pdbg("Cache for '{}' removed.".format(path))
+				if not entry:
+					del cached.cache[absdir]
+					pdbg("Empty directory '{}' in cache also removed.".format(absdir))
+			else:
+				notfound()
+		else:
+			notfound()
+
+	@staticmethod
+	def remove_path_and_cache(path):
+		result = removepath(path)
+		if result == const.ENoError and os.path.isfile(path):
+			cached.remove(path)
+		return result
 
 @cached
 def md5(filename, slice = const.OneM):

bypy/const.py

@@ -12,7 +12,7 @@ import os
 
 # https://packaging.python.org/single_source_version/
 __title__ = 'bypy'
-__version__ = '1.5.7'
+__version__ = '1.5.8'
 __author__ = 'Hou Tianze'
 __license__ = 'MIT'
 __desc__ = 'Python client for Baidu Yun (Personal Cloud Storage) 百度云/百度网盘 Python 客户端'
@@ -45,6 +45,7 @@ EMigrationFailed = 170
 EDownloadCerts = 180
 EUserRejected = 190 # user's decision
 EUpdateNeeded = 200
+ESkipped = 210
 EFatal = -1 # No way to continue
 # internal errors
 IEMD5NotFound = 31079 # File md5 not found, you should use upload API to upload the whole file.
@@ -157,8 +158,8 @@ HomeDir = os.path.expanduser('~')
 ConfigDir = HomeDir + os.sep + '.bypy'
 TokenFileName = 'bypy.json'
 TokenFilePath = ConfigDir + os.sep + TokenFileName
-SettingFileName= 'bypy.setting.json'
-SettingFilePath= ConfigDir + os.sep + SettingFileName
+SettingFileName = 'bypy.setting.json'
+SettingFilePath = ConfigDir + os.sep + SettingFileName
 HashCacheFileName = 'bypy.hashcache.json'
 HashCachePath = ConfigDir + os.sep + HashCacheFileName
 PickleFileName = 'bypy.pickle'
@@ -198,10 +199,10 @@ DefaultResumeDlRevertCount = 1
 DefaultProcessCount = 1
 
 ## program switches
-CleanOptionShort= '-c'
-CleanOptionLong= '--clean'
+CleanOptionShort = '-c'
+CleanOptionLong = '--clean'
 DisableSslCheckOption = '--disable-ssl-check'
 CaCertsOption = '--cacerts'
-MultiprocessOption= '--processes'
+MultiprocessOption = '--processes'
 
 # vim: tabstop=4 noexpandtab shiftwidth=4 softtabstop=4 ff=unix fileencoding=utf-8

bypy/test/configdir/bypy.json

 {
-  "access_token": "21.6cf7a35c11b4a03821ba5167e1c8b3b1.2592000.1499484200.2844184044-1572671", 
+  "access_token": "21.4bdede589a1f3873a115d792e5534339.2592000.1506413842.2844184044-1572671", 
   "expires_in": 2592000, 
-  "refresh_token": "22.2ea6ee64f126b814b28f390e59524bd5.315360000.1812252200.2844184044-1572671", 
+  "refresh_token": "22.6510cc065ac5d13ec0b2ba827ff98925.315360000.1819181842.2844184044-1572671", 
   "scope": "basic netdisk", 
-  "session_key": "9mnRciPjdsLH5WVP7Ui5TXr6nw//f3N8M0o8ZS73gm6wzJYDC4jAs5QXlvIgdejtEicNleO6uKiTHddshZnkn2nTG6H4SR/d4w==", 
-  "session_secret": "fa7e4ae162455a2d494f1b4b3b5283d8"
+  "session_key": "9mnRcFT+MMp/uu69/2WfGd5+f+bg9lQkU9+LhKP8ocGwn+HTsSUWF79CSqzD+UhYyZ6h4lUFJ7wWIGx2pJEkyLPDyQEzrh4CPw==", 
+  "session_secret": "2e32830c3218507fcecc3eed1472fc18"
 }
\ No newline at end of file

bypy/test/configdir/bypy.setting.json

 {
-  "lastUpdateCheck": 1496892192, 
+  "lastUpdateCheck": 1503821836, 
   "overwriteRemoteTempDir": true
 }
\ No newline at end of file

bypy/util.py

@@ -154,6 +154,15 @@ def removedir(path, verbose = False):
 
 	return result
 
+def removepath(path):
+	if os.path.isdir(path):
+		return removedir(path)
+	elif os.path.isfile(path):
+		return removefile(path)
+	else:
+		perr("Can't remove '{}', it's non-file and none-dir.".format(path))
+		return const.EArgument
+
 def makedir(path, mode = 0o777, verbose = False):
 	result = const.ENoError
 

update/update.json

 {
 	"comment": "Update info",
-	"recommendedVersion": "1.5.5",
+	"recommendedVersion": "1.5.8",
 	"minimumRequiredVersion": "1.4.3"
 }