# -*- coding: utf-8 -*-
# Copyright 2011 Google Inc. All Rights Reserved.
# Copyright 2011, Nexenta Systems Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Implementation of Unix-like cp command for cloud storage providers."""
from __future__ import absolute_import
import os
import time
import traceback
from gslib import copy_helper
from gslib.cat_helper import CatHelper
from gslib.command import Command
from gslib.command_argument import CommandArgument
from gslib.commands.compose import MAX_COMPONENT_COUNT
from gslib.copy_helper import CreateCopyHelperOpts
from gslib.copy_helper import ItemExistsError
from gslib.copy_helper import Manifest
from gslib.copy_helper import PARALLEL_UPLOAD_TEMP_NAMESPACE
from gslib.copy_helper import SkipUnsupportedObjectError
from gslib.cs_api_map import ApiSelector
from gslib.exception import CommandException
from gslib.name_expansion import NameExpansionIterator
from gslib.storage_url import ContainsWildcard
from gslib.util import CreateLock
from gslib.util import GetCloudApiInstance
from gslib.util import IsCloudSubdirPlaceholder
from gslib.util import MakeHumanReadable
from gslib.util import NO_MAX
from gslib.util import RemoveCRLFFromString
from gslib.util import StdinIterator
_SYNOPSIS = """
gsutil cp [OPTION]... src_url dst_url
gsutil cp [OPTION]... src_url... dst_url
gsutil cp [OPTION]... -I dst_url
"""
_SYNOPSIS_TEXT = """
<B>SYNOPSIS</B>
""" + _SYNOPSIS
_DESCRIPTION_TEXT = """
<B>DESCRIPTION</B>
The gsutil cp command allows you to copy data between your local file
system and the cloud, copy data within the cloud, and copy data between
cloud storage providers. For example, to copy all text files from the
local directory to a bucket you could do:
gsutil cp *.txt gs://my_bucket
Similarly, you can download text files from a bucket by doing:
gsutil cp gs://my_bucket/*.txt .
If you want to copy an entire directory tree you need to use the -r option:
gsutil cp -r dir gs://my_bucket
If you have a large number of files to upload you might want to use the
gsutil -m option, to perform a parallel (multi-threaded/multi-processing)
copy:
gsutil -m cp -r dir gs://my_bucket
You can pass a list of URLs (one per line) to copy on stdin instead of as
command line arguments by using the -I option. This allows you to use gsutil
in a pipeline to upload or download files / objects as generated by a program,
such as:
some_program | gsutil -m cp -I gs://my_bucket
or:
some_program | gsutil -m cp -I ./download_dir
The contents of stdin can name files, cloud URLs, and wildcards of files
and cloud URLs.
"""
_NAME_CONSTRUCTION_TEXT = """
<B>HOW NAMES ARE CONSTRUCTED</B>
The gsutil cp command strives to name objects in a way consistent with how
Linux cp works, which causes names to be constructed in varying ways depending
on whether you're performing a recursive directory copy or copying
individually named objects; and whether you're copying to an existing or
non-existent directory.
When performing recursive directory copies, object names are constructed
that mirror the source directory structure starting at the point of
recursive processing. For example, the command:
gsutil cp -r dir1/dir2 gs://my_bucket
will create objects named like gs://my_bucket/dir2/a/b/c, assuming
dir1/dir2 contains the file a/b/c.
In contrast, copying individually named files will result in objects named
by the final path component of the source files. For example, the command:
gsutil cp dir1/dir2/** gs://my_bucket
will create objects named like gs://my_bucket/c.
The same rules apply for downloads: recursive copies of buckets and
bucket subdirectories produce a mirrored filename structure, while copying
individually (or wildcard) named objects produce flatly named files.
Note that in the above example the '**' wildcard matches all names
anywhere under dir. The wildcard '*' will match names just one level deep. For
more details see 'gsutil help wildcards'.
There's an additional wrinkle when working with subdirectories: the resulting
names depend on whether the destination subdirectory exists. For example,
if gs://my_bucket/subdir exists as a subdirectory, the command:
gsutil cp -r dir1/dir2 gs://my_bucket/subdir
will create objects named like gs://my_bucket/subdir/dir2/a/b/c. In contrast,
if gs://my_bucket/subdir does not exist, this same gsutil cp command will
create objects named like gs://my_bucket/subdir/a/b/c.
Note: If you use the
`Google Developers Console <https://console.developers.google.com>`_
to create folders, it does so by creating a "placeholder" object that ends
with a "/" character. gsutil skips these objects when downloading from the
cloud to the local file system, because attempting to create a file that
ends with a "/" is not allowed on Linux and MacOS. Because of this, it is
recommended that you not create objects that end with "/" (unless you don't
need to be able to download such objects using gsutil).
"""
_SUBDIRECTORIES_TEXT = """
<B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B>
You can use gsutil to copy to and from subdirectories by using a command
like:
gsutil cp -r dir gs://my_bucket/data
This will cause dir and all of its files and nested subdirectories to be
copied under the specified destination, resulting in objects with names like
gs://my_bucket/data/dir/a/b/c. Similarly you can download from bucket
subdirectories by using a command like:
gsutil cp -r gs://my_bucket/data dir
This will cause everything nested under gs://my_bucket/data to be downloaded
into dir, resulting in files with names like dir/data/a/b/c.
Copying subdirectories is useful if you want to add data to an existing
bucket directory structure over time. It's also useful if you want
to parallelize uploads and downloads across multiple machines (often
reducing overall transfer time compared with simply running gsutil -m
cp on one machine). For example, if your bucket contains this structure:
gs://my_bucket/data/result_set_01/
gs://my_bucket/data/result_set_02/
...
gs://my_bucket/data/result_set_99/
you could perform concurrent downloads across 3 machines by running these
commands on each machine, respectively:
gsutil -m cp -r gs://my_bucket/data/result_set_[0-3]* dir
gsutil -m cp -r gs://my_bucket/data/result_set_[4-6]* dir
gsutil -m cp -r gs://my_bucket/data/result_set_[7-9]* dir
Note that dir could be a local directory on each machine, or it could
be a directory mounted off of a shared file server; whether the latter
performs acceptably may depend on a number of things, so we recommend
you experiment and find out what works best for you.
"""
_COPY_IN_CLOUD_TEXT = """
<B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B>
If both the source and destination URL are cloud URLs from the same
provider, gsutil copies data "in the cloud" (i.e., without downloading
to and uploading from the machine where you run gsutil). In addition to
the performance and cost advantages of doing this, copying in the cloud
preserves metadata (like Content-Type and Cache-Control). In contrast,
when you download data from the cloud it ends up in a file, which has
no associated metadata. Thus, unless you have some way to hold on to
or re-create that metadata, downloading to a file will not retain the
metadata.
Copies spanning locations and/or storage classes cause data to be rewritten
in the cloud, which may take some time. Such operations can be resumed with
the same command if they are interrupted, so long as the command parameters
are identical.
Note that by default, the gsutil cp command does not copy the object
ACL to the new object, and instead will use the default bucket ACL (see
"gsutil help defacl"). You can override this behavior with the -p
option (see OPTIONS below).
One additional note about copying in the cloud: If the destination bucket has
versioning enabled, gsutil cp will by default copy only live versions of the
source object(s). For example:
gsutil cp gs://bucket1/obj gs://bucket2
will cause only the single live version of of gs://bucket1/obj to be copied
to gs://bucket2, even if there are archived versions of gs://bucket1/obj. To
also copy archived versions, use the -A flag:
gsutil cp -A gs://bucket1/obj gs://bucket2
The gsutil -m flag is disallowed when using the cp -A flag, to ensure that
version ordering is preserved.
"""
_CHECKSUM_VALIDATION_TEXT = """
<B>CHECKSUM VALIDATION</B>
At the end of every upload or download the gsutil cp command validates that
the checksum it computes for the source file/object matches the checksum
the service computes. If the checksums do not match, gsutil will delete the
corrupted object and print a warning message. This very rarely happens, but
if it does, please contact gs-team@google.com.
If you know the MD5 of a file before uploading you can specify it in the
Content-MD5 header, which will cause the cloud storage service to reject the
upload if the MD5 doesn't match the value computed by the service. For
example:
% gsutil hash obj
Hashing obj:
Hashes [base64] for obj:
Hash (crc32c): lIMoIw==
Hash (md5): VgyllJgiiaRAbyUUIqDMmw==
% gsutil -h Content-MD5:VgyllJgiiaRAbyUUIqDMmw== cp obj gs://your-bucket/obj
Copying file://obj [Content-Type=text/plain]...
Uploading gs://your-bucket/obj: 182 b/182 B
If the checksum didn't match the service would instead reject the upload and
gsutil would print a message like:
BadRequestException: 400 Provided MD5 hash "VgyllJgiiaRAbyUUIqDMmw=="
doesn't match calculated MD5 hash "7gyllJgiiaRAbyUUIqDMmw==".
Even if you don't do this gsutil will delete the object if the computed
checksum mismatches, but specifying the Content-MD5 header has three
advantages:
1. It prevents the corrupted object from becoming visible at all, whereas
otherwise it would be visible for 1-3 seconds before gsutil deletes it.
2. It will definitively prevent the corrupted object from being left in
the cloud, whereas the gsutil approach of deleting after the upload
completes could fail if (for example) the gsutil process gets ^C'd
between upload and deletion request.
3. It supports a customer-to-service integrity check handoff. For example,
if you have a content production pipeline that generates data to be
uploaded to the cloud along with checksums of that data, specifying the
MD5 computed by your content pipeline when you run gsutil cp will ensure
that the checksums match all the way through the process (e.g., detecting
if data gets corrupted on your local disk between the time it was written
by your content pipeline and the time it was uploaded to GCS).
Note: The Content-MD5 header is ignored for composite objects, because such
objects only have a CRC32C checksum.
"""
_RETRY_HANDLING_TEXT = """
<B>RETRY HANDLING</B>
The cp command will retry when failures occur, but if enough failures happen
during a particular copy or delete operation the command will skip that object
and move on. At the end of the copy run if any failures were not successfully
retried, the cp command will report the count of failures, and exit with
non-zero status.
Note that there are cases where retrying will never succeed, such as if you
don't have write permission to the destination bucket or if the destination
path for some objects is longer than the maximum allowed length.
For more details about gsutil's retry handling, please see
"gsutil help retries".
"""
_RESUMABLE_TRANSFERS_TEXT = """
<B>RESUMABLE TRANSFERS</B>
gsutil automatically uses the Google Cloud Storage resumable upload feature
whenever you use the cp command to upload an object that is larger than 2
MiB. You do not need to specify any special command line options to make this
happen. If your upload is interrupted you can restart the upload by running
the same cp command that you ran to start the upload. Until the upload
has completed successfully, it will not be visible at the destination object
and will not replace any existing object the upload is intended to overwrite.
(However, see the section on PARALLEL COMPOSITE UPLOADS, which may leave
temporary component objects in place during the upload process.)
Similarly, gsutil automatically performs resumable downloads (using HTTP
standard Range GET operations) whenever you use the cp command, unless the
destination is a stream or null. In this case, a partially downloaded
temporary file will be visible in the destination directory. Upon completion,
the original file is deleted and overwritten with the downloaded contents.
Resumable uploads and downloads store some state information in a files
in ~/.gsutil named by the destination object or file. If you attempt to
resume a transfer from a machine with a different directory, the transfer
will start over from scratch.
See also "gsutil help prod" for details on using resumable transfers
in production.
"""
_STREAMING_TRANSFERS_TEXT = """
<B>STREAMING TRANSFERS</B>
Use '-' in place of src_url or dst_url to perform a streaming
transfer. For example:
long_running_computation | gsutil cp - gs://my_bucket/obj
Streaming uploads using the JSON API (see "gsutil help apis") are buffered in
memory and can retry in the event of network flakiness or service errors.
Streaming transfers (other than uploads using the JSON API) do not support
resumable uploads/downloads. If you have a large amount of data to upload
(say, more than 100 MiB) it is recommended to write the data to a local file
and then copy that file to the cloud rather than streaming it (and similarly
for large downloads).
WARNING: When performing streaming transfers gsutil does not compute a
checksum of the uploaded or downloaded data. Therefore, we recommend that
users either perform their own validation of the data or use non-streaming
transfers (which perform integrity checking automatically).
"""
_SLICED_OBJECT_DOWNLOADS_TEXT = """
<B>SLICED OBJECT DOWNLOADS</B>
gsutil automatically uses HTTP Range GET requests to perform "sliced"
downloads in parallel for downloads of large objects. This means that, if
enabled, disk space for the temporary download destination file will be
pre-allocated and byte ranges (slices) within the file will be downloaded in
parallel. Once all slices have completed downloading, the temporary file will
be renamed to the destination file. No additional local disk space is
required for this operation.
This feature is only available for Google Cloud Storage objects because it
requires a fast composable checksum that can be used to verify the data
integrity of the slices. Thus, using sliced object downloads also requires a
compiled crcmod (see "gsutil help crcmod") on the machine performing the
download. If compiled crcmod is not available, normal download will instead
be used.
Note: since sliced object downloads cause multiple writes to occur at various
locations on disk, this can degrade performance for disks with slow seek
times, especially for large numbers of slices. While the default number of
slices is small to avoid this, sliced object download can be completely
disabled by setting the "sliced_object_download_threshold" variable in the
.boto config file to 0.
"""
_PARALLEL_COMPOSITE_UPLOADS_TEXT = """
<B>PARALLEL COMPOSITE UPLOADS</B>
gsutil can automatically use
`object composition <https://developers.google.com/storage/docs/composite-objects>`_
to perform uploads in parallel for large, local files being uploaded to Google
Cloud Storage. This means that, if enabled (see next paragraph), a large file
will be split into component pieces that will be uploaded in parallel. Those
components will then be composed in the cloud, and the temporary components in
the cloud will be deleted after successful composition. No additional local
disk space is required for this operation.
Using parallel composite uploads presents a tradeoff between upload
performance and download configuration: If you enable parallel composite
uploads your uploads will run faster, but someone will need to install a
compiled crcmod (see "gsutil help crcmod") on every machine where objects are
downloaded by gsutil or other Python applications. For some distributions this
is easy (e.g., it comes pre-installed on MacOS), but in some cases users have
found it difficult. Because of this at present parallel composite uploads are
disabled by default. Google is actively working with a number of the Linux
distributions to get crcmod included with the stock distribution. Once that is
done we will re-enable parallel composite uploads by default in gsutil.
Parallel composite uploads should not be used with NEARLINE storage
class buckets, as doing this would incur an early deletion charge for each
component object.
To try parallel composite uploads you can run the command:
gsutil -o GSUtil:parallel_composite_upload_threshold=150M cp bigfile gs://your-bucket
where bigfile is larger than 150 MiB. When you do this notice that the upload
progress indicator continuously updates for several different uploads at once
(corresponding to each of the sections of the file being uploaded in
parallel), until the parallel upload completes. If you then want to enable
parallel composite uploads for all of your future uploads (notwithstanding the
caveats mentioned earlier), you can uncomment and set the
"parallel_composite_upload_threshold" config value in your .boto configuration
file to this value.
Note that the crcmod problem only impacts downloads via Python applications
(such as gsutil). If any users who need to download the data using gsutil or
other Python applications can install crcmod, it makes sense to enable
parallel composite uploads (see above). For example, if you use gsutil to
upload video assets and those assets will only ever be served via a Java
application (there are efficient crc32c implementations available in Java), it
would make sense to enable parallel composite uploads on your machine.
If a parallel composite upload fails prior to composition, re-running the
gsutil command will take advantage of resumable uploads for those components
that failed, and the component objects will be deleted after the first
successful attempt. Any temporary objects that were uploaded successfully
before gsutil failed will still exist until the upload is completed
successfully. The temporary objects will be named in the following fashion:
<random ID>%s<hash>
where <random ID> is some numerical value, and <hash> is an MD5 hash (not
related to the hash of the contents of the file or object).
To avoid leaving temporary objects around, you should make sure to check the
exit status from the gsutil command. This can be done in a bash script, for
example, by doing:
gsutil cp ./local-file gs://your-bucket/your-object
if [ "$status" -ne "0" ] ; then
<< Code that handles failures >>
fi
Or, for copying a directory, use this instead:
gsutil cp -c -L cp.log -r ./dir gs://bucket
if [ "$status" -ne "0" ] ; then
<< Code that handles failures >>
fi
One important caveat is that files uploaded in this fashion are still subject
to the maximum number of components limit. For example, if you upload a large
file that gets split into %d components, and try to compose it with another
object with %d components, the operation will fail because it exceeds the %d
component limit. If you wish to compose an object later and the component
limit is a concern, it is recommended that you disable parallel composite
uploads for that transfer.
Also note that an object uploaded using this feature will have a CRC32C hash,
but it will not have an MD5 hash (and because of that, requires users who
download the object to have crcmod installed, as noted earlier). For details
see 'gsutil help crc32c'.
Note that this feature can be completely disabled by setting the
"parallel_composite_upload_threshold" variable in the .boto config file to 0.
""" % (PARALLEL_UPLOAD_TEMP_NAMESPACE, 10, MAX_COMPONENT_COUNT - 9,
MAX_COMPONENT_COUNT)
_CHANGING_TEMP_DIRECTORIES_TEXT = """
<B>CHANGING TEMP DIRECTORIES</B>
gsutil writes data to a temporary directory in several cases:
- when compressing data to be uploaded (see the -z option)
- when decompressing data being downloaded (when the data has
Content-Encoding:gzip, e.g., as happens when uploaded using gsutil cp -z)
- when running integration tests (using the gsutil test command)
In these cases it's possible the temp file location on your system that
gsutil selects by default may not have enough space. If you find that
gsutil runs out of space during one of these operations (e.g., raising
"CommandException: Inadequate temp space available to compress <your file>"
during a gsutil cp -z operation), you can change where it writes these
temp files by setting the TMPDIR environment variable. On Linux and MacOS
you can do this either by running gsutil this way:
TMPDIR=/some/directory gsutil cp ...
or by adding this line to your ~/.bashrc file and then restarting the shell
before running gsutil:
export TMPDIR=/some/directory
On Windows 7 you can change the TMPDIR environment variable from Start ->
Computer -> System -> Advanced System Settings -> Environment Variables.
You need to reboot after making this change for it to take effect. (Rebooting
is not necessary after running the export command on Linux and MacOS.)
"""
_OPTIONS_TEXT = """
<B>OPTIONS</B>
-a canned_acl Sets named canned_acl when uploaded objects created. See
'gsutil help acls' for further details.
-A Copy all source versions from a source buckets/folders.
If not set, only the live version of each source object is
copied. Note: this option is only useful when the destination
bucket has versioning enabled.
-c If an error occurs, continue to attempt to copy the remaining
files. If any copies were unsuccessful, gsutil's exit status
will be non-zero even if this flag is set. This option is
implicitly set when running "gsutil -m cp...". Note: -c only
applies to the actual copying operation. If an error occurs
while iterating over the files in the local directory (e.g.,
invalid Unicode file name) gsutil will print an error message
and abort.
-D Copy in "daisy chain" mode, i.e., copying between two buckets
by hooking a download to an upload, via the machine where
gsutil is run. By default, data are copied between two buckets
"in the cloud", i.e., without needing to copy via the machine
where gsutil runs.
By default, a "copy in the cloud" when the source is a
composite object will retain the composite nature of the
object. However, Daisy chain mode can be used to change a
composite object into a non-composite object. For example:
gsutil cp -D -p gs://bucket/obj gs://bucket/obj_tmp
gsutil mv -p gs://bucket/obj_tmp gs://bucket/obj
Note: Daisy chain mode is automatically used when copying
between providers (e.g., to copy data from Google Cloud Storage
to another provider).
-e Exclude symlinks. When specified, symbolic links will not be
copied.
-I Causes gsutil to read the list of files or objects to copy from
stdin. This allows you to run a program that generates the list
of files to upload/download.
-L <file> Outputs a manifest log file with detailed information about
each item that was copied. This manifest contains the following
information for each item:
- Source path.
- Destination path.
- Source size.
- Bytes transferred.
- MD5 hash.
- UTC date and time transfer was started in ISO 8601 format.
- UTC date and time transfer was completed in ISO 8601 format.
- Upload id, if a resumable upload was performed.
- Final result of the attempted transfer, success or failure.
- Failure details, if any.
If the log file already exists, gsutil will use the file as an
input to the copy process, and will also append log items to
the existing file. Files/objects that are marked in the
existing log file as having been successfully copied (or
skipped) will be ignored. Files/objects without entries will be
copied and ones previously marked as unsuccessful will be
retried. This can be used in conjunction with the -c option to
build a script that copies a large number of objects reliably,
using a bash script like the following:
until gsutil cp -c -L cp.log -r ./dir gs://bucket; do
sleep 1
done
The -c option will cause copying to continue after failures
occur, and the -L option will allow gsutil to pick up where it
left off without duplicating work. The loop will continue
running as long as gsutil exits with a non-zero status (such a
status indicates there was at least one failure during the
gsutil run).
Note: If you're trying to synchronize the contents of a
directory and a bucket (or two buckets), see
'gsutil help rsync'.
-n No-clobber. When specified, existing files or objects at the
destination will not be overwritten. Any items that are skipped
by this option will be reported as being skipped. This option
will perform an additional GET request to check if an item
exists before attempting to upload the data. This will save
retransmitting data, but the additional HTTP requests may make
small object transfers slower and more expensive.
-p Causes ACLs to be preserved when copying in the cloud. Note
that this option has performance and cost implications when
using the XML API, as it requires separate HTTP calls for
interacting with ACLs. The performance issue can be mitigated
to some degree by using gsutil -m cp to cause parallel copying.
Also, this option only works if you have OWNER access to all of
the objects that are copied.
You can avoid the additional performance and cost of using
cp -p if you want all objects in the destination bucket to end
up with the same ACL by setting a default object ACL on that
bucket instead of using cp -p. See "help gsutil defacl".
Note that it's not valid to specify both the -a and -p options
together.
-R, -r Causes directories, buckets, and bucket subdirectories to be
copied recursively. If you neglect to use this option for
an upload, gsutil will copy any files it finds and skip any
directories. Similarly, neglecting to specify -r for a download
will cause gsutil to copy any objects at the current bucket
directory level, and skip any subdirectories.
-U Skip objects with unsupported object types instead of failing.
Unsupported object types are Amazon S3 Objects in the GLACIER
storage class.
-v Requests that the version-specific URL for each uploaded object
be printed. Given this URL you can make future upload requests
that are safe in the face of concurrent updates, because Google
Cloud Storage will refuse to perform the update if the current
object version doesn't match the version-specific URL. See
'gsutil help versions' for more details.
-z <ext,...> Applies gzip content-encoding to file uploads with the given
extensions. This is useful when uploading files with
compressible content (such as .js, .css, or .html files)
because it saves network bandwidth and space in Google Cloud
Storage, which in turn reduces storage costs.
When you specify the -z option, the data from your files is
compressed before it is uploaded, but your actual files are
left uncompressed on the local disk. The uploaded objects
retain the Content-Type and name of the original files but are
given a Content-Encoding header with the value "gzip" to
indicate that the object data stored are compressed on the
Google Cloud Storage servers.
For example, the following command:
gsutil cp -z html -a public-read cattypes.html gs://mycats
will do all of the following:
- Upload as the object gs://mycats/cattypes.html (cp command)
- Set the Content-Type to text/html (based on file extension)
- Compress the data in the file cattypes.html (-z option)
- Set the Content-Encoding to gzip (-z option)
- Set the ACL to public-read (-a option)
- If a user tries to view cattypes.html in a browser, the
browser will know to uncompress the data based on the
Content-Encoding header, and to render it as HTML based on
the Content-Type header.
Note that if you download an object with Content-Encoding:gzip
gsutil will decompress the content before writing the local
file.
"""
_DETAILED_HELP_TEXT = '\n\n'.join([_SYNOPSIS_TEXT,
_DESCRIPTION_TEXT,
_NAME_CONSTRUCTION_TEXT,
_SUBDIRECTORIES_TEXT,
_COPY_IN_CLOUD_TEXT,
_CHECKSUM_VALIDATION_TEXT,
_RETRY_HANDLING_TEXT,
_RESUMABLE_TRANSFERS_TEXT,
_STREAMING_TRANSFERS_TEXT,
_SLICED_OBJECT_DOWNLOADS_TEXT,
_PARALLEL_COMPOSITE_UPLOADS_TEXT,
_CHANGING_TEMP_DIRECTORIES_TEXT,
_OPTIONS_TEXT])
CP_SUB_ARGS = 'a:AcDeIL:MNnprRtUvz:'
def _CopyFuncWrapper(cls, args, thread_state=None):
cls.CopyFunc(args, thread_state=thread_state)
def _CopyExceptionHandler(cls, e):
"""Simple exception handler to allow post-completion status."""
cls.logger.error(str(e))
cls.op_failure_count += 1
cls.logger.debug('\n\nEncountered exception while copying:\n%s\n',
traceback.format_exc())
def _RmExceptionHandler(cls, e):
"""Simple exception handler to allow post-completion status."""
cls.logger.error(str(e))
class CpCommand(Command):
"""Implementation of gsutil cp command.
Note that CpCommand is run for both gsutil cp and gsutil mv. The latter
happens by MvCommand calling CpCommand and passing the hidden (undocumented)
-M option. This allows the copy and remove needed for each mv to run
together (rather than first running all the cp's and then all the rm's, as
we originally had implemented), which in turn avoids the following problem
with removing the wrong objects: starting with a bucket containing only
the object gs://bucket/obj, say the user does:
gsutil mv gs://bucket/* gs://bucket/d.txt
If we ran all the cp's and then all the rm's and we didn't expand the wildcard
first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt,
and the rm command would then remove that object. In the implementation
prior to gsutil release 3.12 we avoided this by building a list of objects
to process and then running the copies and then the removes; but building
the list up front limits scalability (compared with the current approach
of processing the bucket listing iterator on the fly).
"""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'cp',
command_name_aliases=['copy'],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
# -t is deprecated but leave intact for now to avoid breakage.
supported_sub_args=CP_SUB_ARGS,
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
supported_private_args=['testcallbackfile='],
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument()
]
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='cp',
help_name_aliases=['copy'],
help_type='command_help',
help_one_line_summary='Copy files and objects',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
# pylint: disable=too-many-statements
def CopyFunc(self, name_expansion_result, thread_state=None):
"""Worker function for performing the actual copy (and rm, for mv)."""
gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
copy_helper_opts = copy_helper.GetCopyHelperOpts()
if copy_helper_opts.perform_mv:
cmd_name = 'mv'
else:
cmd_name = self.command_name
src_url = name_expansion_result.source_storage_url
exp_src_url = name_expansion_result.expanded_storage_url
src_url_names_container = name_expansion_result.names_container
have_multiple_srcs = name_expansion_result.is_multi_source_request
if src_url.IsCloudUrl() and src_url.IsProvider():
raise CommandException(
'The %s command does not allow provider-only source URLs (%s)' %
(cmd_name, src_url))
if have_multiple_srcs:
copy_helper.InsistDstUrlNamesContainer(
self.exp_dst_url, self.have_existing_dst_container, cmd_name)
# Various GUI tools (like the GCS web console) create placeholder objects
# ending with '/' when the user creates an empty directory. Normally these
# tools should delete those placeholders once objects have been written
# "under" the directory, but sometimes the placeholders are left around. We
# need to filter them out here, otherwise if the user tries to rsync from
# GCS to a local directory it will result in a directory/file conflict
# (e.g., trying to download an object called "mydata/" where the local
# directory "mydata" exists).
if IsCloudSubdirPlaceholder(exp_src_url):
# We used to output the message 'Skipping cloud sub-directory placeholder
# object...' but we no longer do so because it caused customer confusion.
return
if copy_helper_opts.use_manifest and self.manifest.WasSuccessful(
exp_src_url.url_string):
return
if copy_helper_opts.perform_mv:
if name_expansion_result.names_container:
# Use recursion_requested when performing name expansion for the
# directory mv case so we can determine if any of the source URLs are
# directories (and then use cp -r and rm -r to perform the move, to
# match the behavior of Linux mv (which when moving a directory moves
# all the contained files).
self.recursion_requested = True
# Disallow wildcard src URLs when moving directories, as supporting it
# would make the name transformation too complex and would also be
# dangerous (e.g., someone could accidentally move many objects to the
# wrong name, or accidentally overwrite many objects).
if ContainsWildcard(src_url.url_string):
raise CommandException('The mv command disallows naming source '
'directories using wildcards')
if (self.exp_dst_url.IsFileUrl()
and not os.path.exists(self.exp_dst_url.object_name)
and have_multiple_srcs):
os.makedirs(self.exp_dst_url.object_name)
dst_url = copy_helper.ConstructDstUrl(
src_url, exp_src_url, src_url_names_container, have_multiple_srcs,
self.exp_dst_url, self.have_existing_dst_container,
self.recursion_requested)
dst_url = copy_helper.FixWindowsNaming(src_url, dst_url)
copy_helper.CheckForDirFileConflict(exp_src_url, dst_url)
if copy_helper.SrcDstSame(exp_src_url, dst_url):
raise CommandException('%s: "%s" and "%s" are the same file - '
'abort.' % (cmd_name, exp_src_url, dst_url))
if dst_url.IsCloudUrl() and dst_url.HasGeneration():
raise CommandException('%s: a version-specific URL\n(%s)\ncannot be '
'the destination for gsutil cp - abort.'
% (cmd_name, dst_url))
elapsed_time = bytes_transferred = 0
try:
if copy_helper_opts.use_manifest:
self.manifest.Initialize(
exp_src_url.url_string, dst_url.url_string)
(elapsed_time, bytes_transferred, result_url, md5) = (
copy_helper.PerformCopy(
self.logger, exp_src_url, dst_url, gsutil_api,
self, _CopyExceptionHandler, allow_splitting=True,
headers=self.headers, manifest=self.manifest,
gzip_exts=self.gzip_exts))
if copy_helper_opts.use_manifest:
if md5:
self.manifest.Set(exp_src_url.url_string, 'md5', md5)
self.manifest.SetResult(
exp_src_url.url_string, bytes_transferred, 'OK')
if copy_helper_opts.print_ver:
# Some cases don't return a version-specific URL (e.g., if destination
# is a file).
self.logger.info('Created: %s', result_url)
except ItemExistsError:
message = 'Skipping existing item: %s' % dst_url
self.logger.info(message)
if copy_helper_opts.use_manifest:
self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message)
except SkipUnsupportedObjectError, e:
message = ('Skipping item %s with unsupported object type %s' %
(exp_src_url.url_string, e.unsupported_type))
self.logger.info(message)
if copy_helper_opts.use_manifest:
self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message)
except copy_helper.FileConcurrencySkipError, e:
self.logger.warn('Skipping copy of source URL %s because destination URL '
'%s is already being copied by another gsutil process '
'or thread (did you specify the same source URL twice?) '
% (src_url, dst_url))
except Exception, e:
if (copy_helper_opts.no_clobber and
copy_helper.IsNoClobberServerException(e)):
message = 'Rejected (noclobber): %s' % dst_url
self.logger.info(message)
if copy_helper_opts.use_manifest:
self.manifest.SetResult(
exp_src_url.url_string, 0, 'skip', message)
elif self.continue_on_error:
message = 'Error copying %s: %s' % (src_url, str(e))
self.op_failure_count += 1
self.logger.error(message)
if copy_helper_opts.use_manifest:
self.manifest.SetResult(
exp_src_url.url_string, 0, 'error',
RemoveCRLFFromString(message))
else:
if copy_helper_opts.use_manifest:
self.manifest.SetResult(
exp_src_url.url_string, 0, 'error', str(e))
raise
else:
if copy_helper_opts.perform_mv:
self.logger.info('Removing %s...', exp_src_url)
if exp_src_url.IsCloudUrl():
gsutil_api.DeleteObject(exp_src_url.bucket_name,
exp_src_url.object_name,
generation=exp_src_url.generation,
provider=exp_src_url.scheme)
else:
os.unlink(exp_src_url.object_name)
with self.stats_lock:
self.total_elapsed_time += elapsed_time
self.total_bytes_transferred += bytes_transferred
# Command entry point.
def RunCommand(self):
copy_helper_opts = self._ParseOpts()
self.total_elapsed_time = self.total_bytes_transferred = 0
if self.args[-1] == '-' or self.args[-1] == 'file://-':
return CatHelper(self).CatUrlStrings(self.args[:-1])
if copy_helper_opts.read_args_from_stdin:
if len(self.args) != 1:
raise CommandException('Source URLs cannot be specified with -I option')
url_strs = StdinIterator()
else:
if len(self.args) < 2:
raise CommandException('Wrong number of arguments for "cp" command.')
url_strs = self.args[:-1]
(self.exp_dst_url, self.have_existing_dst_container) = (
copy_helper.ExpandUrlToSingleBlr(self.args[-1], self.gsutil_api,
self.debug, self.project_id))
name_expansion_iterator = NameExpansionIterator(
self.command_name, self.debug,
self.logger, self.gsutil_api, url_strs,
self.recursion_requested or copy_helper_opts.perform_mv,
project_id=self.project_id, all_versions=self.all_versions,
continue_on_error=self.continue_on_error or self.parallel_operations)
# Use a lock to ensure accurate statistics in the face of
# multi-threading/multi-processing.
self.stats_lock = CreateLock()
# Tracks if any copies failed.
self.op_failure_count = 0
# Start the clock.
start_time = time.time()
# Tuple of attributes to share/manage across multiple processes in
# parallel (-m) mode.
shared_attrs = ('op_failure_count', 'total_bytes_transferred')
# Perform copy requests in parallel (-m) mode, if requested, using
# configured number of parallel processes and threads. Otherwise,
# perform requests with sequential function calls in current process.
self.Apply(_CopyFuncWrapper, name_expansion_iterator,
_CopyExceptionHandler, shared_attrs,
fail_on_error=(not self.continue_on_error))
self.logger.debug(
'total_bytes_transferred: %d', self.total_bytes_transferred)
end_time = time.time()
self.total_elapsed_time = end_time - start_time
# Sometimes, particularly when running unit tests, the total elapsed time
# is really small. On Windows, the timer resolution is too small and
# causes total_elapsed_time to be zero.
try:
float(self.total_bytes_transferred) / float(self.total_elapsed_time)
except ZeroDivisionError:
self.total_elapsed_time = 0.01
self.total_bytes_per_second = (float(self.total_bytes_transferred) /
float(self.total_elapsed_time))
if self.debug == 3:
# Note that this only counts the actual GET and PUT bytes for the copy
# - not any transfers for doing wildcard expansion, the initial
# HEAD/GET request performed to get the object metadata, etc.
if self.total_bytes_transferred != 0:
self.logger.info(
'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)',
self.total_bytes_transferred, self.total_elapsed_time,
MakeHumanReadable(self.total_bytes_per_second))
if self.op_failure_count:
plural_str = 's' if self.op_failure_count > 1 else ''
raise CommandException('%d file%s/object%s could not be transferred.' % (
self.op_failure_count, plural_str, plural_str))
return 0
def _ParseOpts(self):
perform_mv = False
# exclude_symlinks is handled by Command parent class, so save in Command
# state rather than CopyHelperOpts.
self.exclude_symlinks = False
no_clobber = False
# continue_on_error is handled by Command parent class, so save in Command
# state rather than CopyHelperOpts.
self.continue_on_error = False
daisy_chain = False
read_args_from_stdin = False
print_ver = False
use_manifest = False
preserve_acl = False
canned_acl = None
# canned_acl is handled by a helper function in parent
# Command class, so save in Command state rather than CopyHelperOpts.
self.canned = None
self.all_versions = False
self.skip_unsupported_objects = False
# Files matching these extensions should be gzipped before uploading.
self.gzip_exts = []
test_callback_file = None
# self.recursion_requested initialized in command.py (so can be checked
# in parent class for all commands).
self.manifest = None
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-a':
canned_acl = a
self.canned = True
if o == '-A':
self.all_versions = True
if o == '-c':
self.continue_on_error = True
elif o == '-D':
daisy_chain = True
elif o == '-e':
self.exclude_symlinks = True
elif o == '--testcallbackfile':
# File path of a pickled class that implements ProgressCallback.call.
# Used for testing transfer interruptions and resumes.
test_callback_file = a
elif o == '-I':
read_args_from_stdin = True
elif o == '-L':
use_manifest = True
self.manifest = Manifest(a)
elif o == '-M':
# Note that we signal to the cp command to perform a move (copy
# followed by remove) and use directory-move naming rules by passing
# the undocumented (for internal use) -M option when running the cp
# command from mv.py.
perform_mv = True
elif o == '-n':
no_clobber = True
elif o == '-p':
preserve_acl = True
elif o == '-r' or o == '-R':
self.recursion_requested = True
elif o == '-U':
self.skip_unsupported_objects = True
elif o == '-v':
print_ver = True
elif o == '-z':
self.gzip_exts = [x.strip() for x in a.split(',')]
if preserve_acl and canned_acl:
raise CommandException(
'Specifying both the -p and -a options together is invalid.')
if self.all_versions and self.parallel_operations:
raise CommandException(
'The gsutil -m option is not supported with the cp -A flag, to '
'ensure that object version ordering is preserved. Please re-run '
'the command without the -m option.')
return CreateCopyHelperOpts(
perform_mv=perform_mv,
no_clobber=no_clobber,
daisy_chain=daisy_chain,
read_args_from_stdin=read_args_from_stdin,
print_ver=print_ver,
use_manifest=use_manifest,
preserve_acl=preserve_acl,
canned_acl=canned_acl,
skip_unsupported_objects=self.skip_unsupported_objects,
test_callback_file=test_callback_file)