snapshot_stream_writer.cc
imports tsl/platform/mutex.himports tsl/platform/path.h
SHA-256ab3f72e4f41ef150f642d1abe73ea94331ce6c8f95fc29d698c686865edd6bc8
MaleculeH(Db)Md(In₄)
Evidence
1/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
2
3Licensed under the Apache License, Version 2.0 (the "License");
4you may not use this file except in compliance with the License.
5You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9Unless required by applicable law or agreed to in writing, software
10distributed under the License is …
11:42… IND, either express or implied.
12See the License for the specific language governing permissions and
13limitations under the License.
14==============================================================================*/
15#include "tensorflow/core/data/service/snapshot/snapshot_stream_writer.h"
16
17#include <algorithm>
18#include <cstddef>
19#include <cstdint>
20#include <memory>
21#include <optional>
22#include <string>
23#include <utility>
24#include <vector>
25
26#include "absl/log/check.h"
27#include "absl/log/log.h"
28#include "absl/memory/memory.h"
29#include "absl/status/status.h"
30#include "absl/status/statusor.h"
31#include "absl/strings/numbers.h"
32#include "absl/strings/str_cat.h"
33#include "absl/strings/str_split.h"
34#include "absl/time/time.h"
35#include "xla/tsl/platform/env.h"
36#include "xla/tsl/platform/statusor.h"
37#include "tensorflow/core/data/service/byte_size.h"
38#include "tensorflow/core/data/service/common.h"
39#include "tensorflow/core/data/service/snapshot/file_utils.h"
40#include "tensorflow/core/data/service/snapshot/parallel_tfrecord_writer.h"
41#include "tensorflow/core/data/service/snapshot/path_utils.h"
42#include "tensorflow/core/data/service/snapshot/utils.h"
43#include "tensorflow/core/data/service/worker.pb.h"
44#include "tensorflow/core/data/snapshot_utils.h"
45#include "tensorflow/core/data/utils.h"
46#include "tensorflow/core/framework/metrics.h"
47#include "tensorflow/core/framework/tensor.h"
48#include "tensorflow/core/framework/tensor.pb.h"
49#include "tensorflow/core/framework/types.pb.h"
50#include "tsl/platform/mutex.h"
51#include "tsl/platform/path.h"
52#include "tsl/profiler/lib/traceme.h"
53
54namespace tensorflow {
⋯4 lines
58:68… );
59constexpr int64_t kUnknownNumElements = -1;
60
61constexpr const char kFileShardDelimiter[] = "_CHUNK_SHARDS_";
62
63// Extracts the index from the `filename` of an uncommitted chunk. The file name
64// is expected to be chu …