diff --git a/README.md b/README.md index 8dd6c0e5a943056e7d5c9cb1a616c775c50125cc..2450dfba236ad83c44daa714b1f3d81bb2696cc5 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ class LogWriter(logdir=None, filename_suffix='', write_to_disk=True, display_name='', + file_name='', **kwargs) ``` @@ -110,6 +111,7 @@ class LogWriter(logdir=None, | filename_suffix | string | 为默认的日志文件名添加后缀 | | write_to_disk | boolean | 是否写入到磁盘 | | display_name | string | 在面板中替换实际显示的`logdir`,当日志所在路径过长或想隐藏日志所在路径时可指定此参数 | +| file_name | string | 指定写入的日志文件名,如果指定的文件名已经存在,则将日志续写在此文件中,文件名必须包括`vdlrecords` | #### 示例 diff --git a/visualdl/io/bfile.py b/visualdl/io/bfile.py index b13284524644501ac42f7cfbf5ee65399e8861d6..baeb8f1b0b6a36e12545596b105955a169cacb20 100644 --- a/visualdl/io/bfile.py +++ b/visualdl/io/bfile.py @@ -130,7 +130,8 @@ class LocalFileSystem(object): self._write(filename, file_content, "ab" if binary_mode else "a") def write(self, filename, file_content, binary_mode=False): - self._write(filename, file_content, "wb" if binary_mode else "w") + self._write(filename, file_content, "ab" if binary_mode else "a") + # self._write(filename, file_content, "wb" if binary_mode else "w") def walk(self, dir): if 'posix' == os.name: @@ -187,7 +188,8 @@ class HDFileSystem(object): self.cli.write(hdfs_path=filename[7:], data=file_content, append=True) def write(self, filename, file_content, binary_mode=False): - self.cli.write(hdfs_path=filename[7:], data=file_content) + self.cli.write(hdfs_path=filename[7:], data=file_content, append=True) + # self.cli.write(hdfs_path=filename[7:], data=file_content) def walk(self, dir): walks = self.cli.walk(hdfs_path=dir[7:]) @@ -326,13 +328,15 @@ class BosFileSystem(object): self._start_append_time = time.time() def write(self, filename, file_content, binary_mode=False): - bucket_name, object_key = BosFileSystem._get_object_info(filename) - - self.bos_client.append_object(bucket_name=bucket_name, - key=object_key, - data=file_content, - content_md5=content_md5(file_content), - content_length=len(file_content)) + self.append(filename, file_content, binary_mode=False) + + # bucket_name, object_key = BosFileSystem._get_object_info(filename) + # + # self.bos_client.append_object(bucket_name=bucket_name, + # key=object_key, + # data=file_content, + # content_md5=content_md5(file_content), + # content_length=len(file_content)) def walk(self, dir): class WalkGenerator(): diff --git a/visualdl/writer/record_writer.py b/visualdl/writer/record_writer.py index 0a3a9b30d06ede6dc5b8b110e5e6ab5c00834882..e80e0ff903ad8913c121a1149e27ea898e79009a 100644 --- a/visualdl/writer/record_writer.py +++ b/visualdl/writer/record_writer.py @@ -74,12 +74,28 @@ class RecordFileWriter(object): file. """ def __init__(self, logdir, max_queue_size=10, flush_secs=120, - filename_suffix=''): + filename_suffix='', filename=''): self._logdir = logdir if not bfile.exists(logdir): bfile.makedirs(logdir) - self._file_name = bfile.join(logdir, "vdlrecords.%010d.log%s" % ( - time.time(), filename_suffix)) + + if filename: + if 'vdlrecords' in filename: + self._file_name = bfile.join(logdir, filename) + if bfile.exists(self._file_name): + print( + '`{}` is exists, VisualDL will add logs to it.'.format( + self._file_name)) + else: + fn = "vdlrecords.%010d.log%s" % (time.time(), filename_suffix) + self._file_name = bfile.join(logdir, fn) + print( + 'Since the log filename should contain `vdlrecords`, the filename is invalid and `{}` will replace `{}`'.format( + fn, filename)) + else: + self._file_name = bfile.join(logdir, "vdlrecords.%010d.log%s" % ( + time.time(), filename_suffix)) + self._general_file_writer = bfile.BFile(self._file_name, "wb") self._async_writer = _AsyncWriter(RecordWriter( self._general_file_writer), max_queue_size, flush_secs) @@ -91,6 +107,9 @@ class RecordFileWriter(object): def get_logdir(self): return self._logdir + def get_filename(self): + return self._file_name + def add_record(self, record): if not isinstance(record, record_pb2.Record): raise TypeError("Expected an record_pb2.Record proto, " diff --git a/visualdl/writer/writer.py b/visualdl/writer/writer.py index 351d4866247b7d4043ca23540a61c9349071bba1..fed40d59c3424255551c457cd76efebf0ea6f86c 100644 --- a/visualdl/writer/writer.py +++ b/visualdl/writer/writer.py @@ -67,6 +67,7 @@ class LogWriter(object): filename_suffix='', write_to_disk=True, display_name='', + file_name='', **kwargs): """Create a instance of class `LogWriter` and create a vdl log file with given args. @@ -94,6 +95,7 @@ class LogWriter(object): self._filename_suffix = filename_suffix self._write_to_disk = write_to_disk self.kwargs = kwargs + self._file_name = file_name self._file_writer = None self._all_writers = {} @@ -116,10 +118,15 @@ class LogWriter(object): logdir=self._logdir, max_queue_size=self._max_queue, flush_secs=self._flush_secs, - filename_suffix=self._filename_suffix) + filename_suffix=self._filename_suffix, + filename=self._file_name) self._all_writers.update({self._logdir: self._file_writer}) return self._file_writer + @property + def file_name(self): + return self._file_writer.get_filename() + def add_meta(self, tag='meta_data_tag', display_name='', step=0, walltime=None): """Add a meta to vdl record file.