Fix #285: list() capped at 1000 files maximum per directory

bypy/bypy.py

@@ -689,7 +689,7 @@ class ByPy(object):
 					self.pd("206 Partial Content (this is OK), processing action")
 				result = act(r, actargs)
 				if result == const.ENoError:
-					self.pd("Request OK.")
+					self.pd("Request OK")
 			else:
 				ec = self.__get_json_errorcode(r, act)
 				#   6 (sc: 403): No permission to access user data
@@ -2144,16 +2144,22 @@ To stream a file, you can use the 'mkfifo' trick with omxplayer etc.:
 		return const.ENoError
 
 	def __walk_remote_dir(self, remotepath, remoterootpath, args = None, skip_remote_only_dirs = False):
-		pars = {
-			'method' : 'list',
-			'path' : remotepath,
-			'by' : 'name',
-			'order' : 'asc' }
-
-		# Python parameters are by-reference and mutable, so they are 'out' by default
 		dirjs = []
 		filejs = []
-		result = self.__get(pcsurl + 'file', pars, self.__walk_proceed_remote_dir_act, (dirjs, filejs))
+		listStart = 0
+		# https://github.com/houtianze/bypy/issues/285
+		while True:
+			pars = {
+				'method' : 'list',
+				'path' : remotepath,
+				'by' : 'name',
+				'order' : 'asc',
+				'limit' : '{}-{}'.format(listStart, listStart + const.MaxListEntries)}
+			# Python parameters are by-reference and mutable, so they are 'out' by default
+			result = self.__get(pcsurl + 'file', pars, self.__walk_proceed_remote_dir_act, (dirjs, filejs))
+			if len(dirjs) + len(filejs) - listStart < const.MaxListEntries:
+				break
+			listStart += const.MaxListEntries
 		yield [result, remotepath, dirjs, filejs, args]
 		if result == const.ENoError:
 			self.pd("Remote dirs: {}".format(dirjs))

bypy/const.py

@@ -159,6 +159,7 @@ DPcsUrl = 'https://d.pcs.baidu.com/rest/2.0/pcs/'
 MinRapidUploadFileSize = 256 * OneK
 MaxSliceSize = 2 * OneG
 MaxSlicePieces = 1024
+MaxListEntries = 1000 # https://github.com/houtianze/bypy/issues/285
 
 ### Auth servers
 GaeUrl = 'https://bypyoauth.appspot.com'

bypy/test/configdir/bypy.setting.json

 {
-  "lastUpdateCheck": 1516271227, 
+  "lastUpdateCheck": 1520267575, 
   "overwriteRemoteTempDir": true
 }
\ No newline at end of file

bypy/test/test.py

@@ -15,6 +15,7 @@ from __future__ import division
 import sys
 import os
 import shutil
+import tempfile
 import re
 import pprint
 import copy
@@ -197,10 +198,10 @@ def assertsame(by):
 	assert len(by.result['remote']) == 0
 	assert len(by.result['same']) >= 5
 
-def compare(by):
+def compare(by, left = TestDir, right = TestDir):
 	# comparison
 	banner("Comparing")
-	assert by.compare(TestDir, TestDir) == const.ENoError
+	assert by.compare(left, right) == const.ENoError
 	assertsame(by)
 	mpr.empty()
 
@@ -274,6 +275,28 @@ def testshare(by):
 	assertsingle(by, filterregex(mpr.getq(), r"bypy accept /{}/subdir/1M2.bin".format(ShareDir)))
 	mpr.empty()
 
+def createmanyfiles(dir, numFiles):
+	shutil.rmtree(dir, ignore_errors=True)
+	os.makedirs(dir)
+	for i in range(0, numFiles):
+		fname = str(i).zfill(4)
+		ffname = os.path.join(dir, fname)
+		with open(ffname, 'w') as f:
+			f.write(fname)
+
+def testmanyfiles(by):
+	pass
+
+# def testmanyfiles(by):
+# 	numFiles = const.MaxListEntries * 2 + 10
+# 	banner("Test uploading of many ({}) files".format(numFiles))
+# 	with tempfile.TemporaryDirectory(prefix = 'bypytest_') as tmpdir:
+# 		print("Testing temp dir: ", tmpdir)
+# 		createmanyfiles(tmpdir, numFiles)
+# 		by.upload(tmpdir, tmpdir)
+# 		compare(by, tmpdir, tmpdir)
+# 	mpr.empty()
+
 def cleanup():
 	os.remove(zerofilename)
 	#shutil.rmtree(ConfigDir, ignore_errors=True)
@@ -309,6 +332,11 @@ def main():
 		by = bypy.ByPy(configdir=ConfigDir, debug=1, verbose=1)
 		if 'refresh' in sys.argv:
 			by.refreshtoken()
+
+		if 'many' in sys.argv:
+			testmanyfiles(by)
+			return
+
 		runTests(by)
 		if '1' in sys.argv:
 			return