From 6d176df02928609364c50631748a5a2e69377dbd Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Wed, 29 Apr 2015 16:46:46 -0700 Subject: [PATCH 01/10] Upgrade GCS sample to v1 Pull up api client dependencies to associated 1.20.0 version and update Guava to latest, 18.0. --- storage-cmdline-sample/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage-cmdline-sample/pom.xml b/storage-cmdline-sample/pom.xml index d26626ac..dbe3e250 100644 --- a/storage-cmdline-sample/pom.xml +++ b/storage-cmdline-sample/pom.xml @@ -88,7 +88,7 @@ com.google.guava guava - 14.0.1 + 18.0 com.google.oauth-client @@ -98,7 +98,7 @@ com.google.apis google-api-services-storage - v1beta2-rev51-1.19.0 + v1-rev33-1.20.0 com.google.http-client @@ -109,9 +109,9 @@ UTF-8 - 1.19.0 - 1.19.0 - 1.19.0 + 1.20.0 + 1.20.0 + 1.20.0 From a1d9b48397522c5f1df6218bc5bcee0a7c60fe90 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Fri, 1 May 2015 16:39:26 -0700 Subject: [PATCH 02/10] Refactor storage sample * Break up the monolithic file into individual standalone examples that can be executed separately. * Move some helper classes into separate files for brevity. * Remove some of the complexity to avoid barriers to entry. --- .../samples/storage/cmdline/Helpers.java | 76 +++ .../storage/cmdline/SampleSettings.java | 57 ++ .../storage/cmdline/StorageSample.java | 516 +++--------------- .../samples/storage/cmdline/View.java | 59 ++ .../storage/examples/BucketsGetExample.java | 49 ++ .../examples/BucketsInsertExample.java | 69 +++ .../examples/ObjectsDownloadExample.java | 70 +++ .../storage/examples/ObjectsGetExample.java | 50 ++ .../storage/examples/ObjectsListExample.java | 58 ++ .../examples/ObjectsUploadExample.java | 78 +++ .../storage/util/CredentialsProvider.java | 79 +++ 11 files changed, 711 insertions(+), 450 deletions(-) create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/SampleSettings.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/View.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsGetExample.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java create mode 100644 storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java new file mode 100644 index 00000000..1aaa1af1 --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.cmdline; + +import java.io.InputStream; +import java.util.Random; + +public class Helpers { + + /** + * Generates a random data block and repeats it to provide the stream. + * + * Using a buffer instead of just filling from java.util.Random because the latter causes + * noticeable lag in stream reading, which detracts from upload speed. This class takes all that + * cost in the constructor. + */ + public static class RandomDataBlockInputStream extends InputStream { + + private long byteCountRemaining; + private final byte[] buffer; + + public RandomDataBlockInputStream(long size, int blockSize) { + byteCountRemaining = size; + final Random random = new Random(); + buffer = new byte[blockSize]; + random.nextBytes(buffer); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read() + */ + @Override + public int read() { + throw new AssertionError("Not implemented; too slow."); + } + + /* + * (non-Javadoc) + * + * @see java.io.InputStream#read(byte [], int, int) + */ + @Override + public int read(byte b[], int off, int len) { + if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } else if (byteCountRemaining == 0) { + return -1; + } + int actualLen = len > byteCountRemaining ? (int) byteCountRemaining : len; + for (int i = off; i < actualLen; i++) { + b[i] = buffer[i % buffer.length]; + } + byteCountRemaining -= actualLen; + return actualLen; + } + } + +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/SampleSettings.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/SampleSettings.java new file mode 100644 index 00000000..0f5c2487 --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/SampleSettings.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.cmdline; + +import com.google.api.client.json.GenericJson; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.util.Key; + +import java.io.IOException; + +/** Samples settings JSON Model. */ +public final class SampleSettings extends GenericJson { + + @Key("project") + private String project; + + @Key("bucket") + private String bucket; + + @Key("prefix") + private String prefix; + + public String getProject() { + return project; + } + + public String getBucket() { + return bucket; + } + + public String getPrefix() { + return prefix; + } + + public static SampleSettings load(JsonFactory jsonFactory) throws IOException { + try { + return jsonFactory.fromInputStream( + StorageSample.class.getResourceAsStream("/sample_settings.json"), SampleSettings.class); + } catch (IOException e) { + IOException e2 = new IOException("Unable to read sample_settings.json: " + e.getMessage(), e); + throw e2; + } + } +} + diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java index 808f0eae..35676c08 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java @@ -14,226 +14,99 @@ package com.google.api.services.samples.storage.cmdline; -import static java.net.HttpURLConnection.HTTP_CONFLICT; - import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; -import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.googleapis.json.GoogleJsonError; import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.client.googleapis.media.MediaHttpDownloader; -import com.google.api.client.googleapis.media.MediaHttpDownloaderProgressListener; -import com.google.api.client.googleapis.media.MediaHttpUploader; -import com.google.api.client.googleapis.media.MediaHttpUploaderProgressListener; -import com.google.api.client.http.HttpHeaders; import com.google.api.client.http.HttpResponseException; import com.google.api.client.http.HttpTransport; -import com.google.api.client.http.InputStreamContent; -import com.google.api.client.json.GenericJson; import com.google.api.client.json.JsonFactory; import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.client.util.Key; -import com.google.api.client.util.Lists; -import com.google.api.client.util.store.DataStoreFactory; -import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.api.services.samples.storage.examples.BucketsGetExample; +import com.google.api.services.samples.storage.examples.BucketsInsertExample; +import com.google.api.services.samples.storage.examples.ObjectsDownloadExample; +import com.google.api.services.samples.storage.examples.ObjectsGetExample; +import com.google.api.services.samples.storage.examples.ObjectsListExample; +import com.google.api.services.samples.storage.examples.ObjectsUploadExample; +import com.google.api.services.samples.storage.util.CredentialsProvider; import com.google.api.services.storage.Storage; -import com.google.api.services.storage.StorageScopes; import com.google.api.services.storage.model.Bucket; -import com.google.api.services.storage.model.ObjectAccessControl; -import com.google.api.services.storage.model.Objects; import com.google.api.services.storage.model.StorageObject; -import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableMap; +import com.google.common.io.BaseEncoding; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Random; +import java.io.OutputStream; +import java.security.DigestOutputStream; +import java.security.MessageDigest; -/** - * @author nherring@google.com (Nathan Herring) - */ public class StorageSample { /** * Be sure to specify the name of your application. If the application name is {@code null} or * blank, the application will log a warning. Suggested format is "MyCompany-ProductName/1.0". */ - private static final String APPLICATION_NAME = ""; - - /** - * Whether or not we're running on AppEngine. Not set for this command-line sample, but is - * relevant if you copy this code for use there. - */ - private static final boolean IS_APP_ENGINE = false; - - /** Directory to store user credentials. */ - private static final java.io.File DATA_STORE_DIR = - new java.io.File(System.getProperty("user.home"), ".store/storage_sample"); - - /** - * Global instance of the {@link DataStoreFactory}. The best practice is to make it a single - * globally shared instance across your application. - */ - private static FileDataStoreFactory dataStoreFactory; - - /** Global instance of the HTTP transport. */ - private static HttpTransport httpTransport; - - /** Global instance of the JSON factory. */ - private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); - - /** Global instance of this sample's settings. */ - private static SampleSettings settings; - - private static Storage storage; - - - /** Authorizes the installed application to access user's protected data. */ - private static Credential authorize() throws Exception { - // load client secrets - GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, - new InputStreamReader(StorageSample.class.getResourceAsStream("/client_secrets.json"))); - if (clientSecrets.getDetails().getClientId().startsWith("Enter") - || clientSecrets.getDetails().getClientSecret().startsWith("Enter ")) { - System.out.println( - "Enter Client ID and Secret from https://code.google.com/apis/console/?api=storage_api " - + "into storage-cmdline-sample/src/main/resources/client_secrets.json"); - System.exit(1); - } - // set up authorization code flow - GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( - httpTransport, JSON_FACTORY, clientSecrets, - Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)).setDataStoreFactory( - dataStoreFactory).build(); - // authorize - return new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()).authorize("user"); - } - - /** Loads sample-specific settings from a JSON file. */ - private static void readSettings() { - settings = SampleSettings.load( - JSON_FACTORY, StorageSample.class.getResourceAsStream("/sample_settings.json")); - if (settings.getProject().startsWith("Enter ") || settings.getBucket().startsWith("Enter ")) { - System.out.println("Enter sample settings into " - + "storage-cmdline-sample/src/main/resources/sample_settings.json"); - System.exit(1); - } - } - - /** Samples settings JSON Model. */ - public static final class SampleSettings extends GenericJson { - @Key("project") - private String project; - - @Key("bucket") - private String bucket; - - @Key("prefix") - private String prefix; - - @Key("email") - private String email; - - @Key("domain") - private String domain; - - public String getProject() { - return project; - } - - public String getBucket() { - return bucket; - } - - public String getPrefix() { - return prefix; - } - - public String getEmail() { - return email; - } - - public String getDomain() { - return domain; - } - - public static SampleSettings load(JsonFactory jsonFactory, InputStream inputStream) { - try { - return jsonFactory.fromInputStream(inputStream, SampleSettings.class); - } catch (IOException e) { - return new SampleSettings(); - } - } - } - - /** Provides simple cmdline UI. */ - public static class View { - - static void header1(String name) { - System.out.println(); - System.out.println("================== " + name + " =================="); - System.out.println(); - } - - static void header2(String name) { - System.out.println(); - System.out.println("~~~~~~~~~~~~~~~~~~ " + name + " ~~~~~~~~~~~~~~~~~~"); - System.out.println(); - } - - static void show(Bucket bucket) { - System.out.println("name: " + bucket.getName()); - System.out.println("location: " + bucket.getLocation()); - System.out.println("timeCreated: " + bucket.getTimeCreated()); - System.out.println("owner: " + bucket.getOwner()); - System.out.println("acl: " + bucket.getAcl()); - } - - static void show(StorageObject object) { - System.out.println("name: " + object.getName()); - System.out.println("size: " + object.getSize()); - System.out.println("contentType: " + object.getContentType()); - System.out.println("updated: " + object.getUpdated()); - System.out.println("owner: " + object.getOwner()); - // should only show up if projection is full. - // System.out.println("acl: " + object.getAcl()); - } - - static void separator() { - System.out.println(); - System.out.println("------------------------------------------------------"); - System.out.println(); - } - } + private static final String APPLICATION_NAME = "Google-StorageSample/1.1"; public static void main(String[] args) { try { - httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR); - // settings - readSettings(); - // authorization - Credential credential = authorize(); - // set up global Storage instance - storage = new Storage.Builder(httpTransport, JSON_FACTORY, credential).setApplicationName( - APPLICATION_NAME).build(); + // initialize network, sample settings, credentials, and the storage client. + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); + SampleSettings settings = SampleSettings.load(JSON_FACTORY); + Credential credential = CredentialsProvider.authorize(httpTransport, JSON_FACTORY); + Storage storage = new Storage.Builder(httpTransport, JSON_FACTORY, credential) + .setApplicationName(APPLICATION_NAME).build(); + + // // run commands - tryCreateBucket(); - getBucket(); - listObjects(); - getObjectMetadata(); - uploadObject(true /* useCustomMetadata */); - getObjectData(); - getPartialObjectData(); + // + View.header1("Trying to create a new bucket " + settings.getBucket()); + BucketsInsertExample.insertInNamedProject(storage, settings.getProject(), + new Bucket().setName(settings.getBucket()).setLocation("US")); + + View.header1("Getting bucket " + settings.getBucket() + " metadata"); + Bucket bucket = BucketsGetExample.get(storage, settings.getBucket()); + View.show(bucket); + + View.header1("Listing objects in bucket " + settings.getBucket()); + for (StorageObject object : ObjectsListExample.list(storage, settings.getBucket())) { + View.show(object); + } + + View.header1("Getting object metadata from gs://pub/SomeOfTheTeam.jpg"); + StorageObject object = ObjectsGetExample.get(storage, "pub", "SomeOfTheTeam.jpg"); + View.show(object); + + View.header1("Uploading object."); + final long objectSize = 100 * 1000 * 1000 /* 100 MB */; + InputStream data = new Helpers.RandomDataBlockInputStream(objectSize, 1024); + object = new StorageObject() + .setBucket(settings.getBucket()) + .setName(settings.getPrefix() + "myobject") + .setMetadata(ImmutableMap.of("key1", "value1", "key2", "value2")) + .setCacheControl("max-age=3600, must-revalidate") + .setContentDisposition("attachment"); + object = ObjectsUploadExample.uploadWithMetadata(storage, object, data); + View.show(object); + System.out.println("md5Hash: " + object.getMd5Hash()); + + View.header1("Getting object data of uploaded object, calculate hashes/crcs."); + OutputStream nullOutputStream = new OutputStream() { + // Throws away the bytes. + @Override public void write(int b) throws IOException {} + @Override public void write(byte b[], int off, int len) {} + }; + DigestOutputStream md5DigestOutputStream = new DigestOutputStream( + nullOutputStream, MessageDigest.getInstance("MD5")); + ObjectsDownloadExample.downloadToOutputStream( + storage, settings.getBucket(), settings.getPrefix() + "myobject", md5DigestOutputStream); + String calculatedMD5 = BaseEncoding.base64().encode(md5DigestOutputStream.getMessageDigest().digest()); + System.out.println("md5Hash: " + calculatedMD5 + " " + (object.getMd5Hash().equals(calculatedMD5) + ? "(MATCHES)" : "(MISMATCHES; data altered in transit)")); + // success! return; } catch (GoogleJsonResponseException e) { @@ -254,261 +127,4 @@ public static void main(String[] args) { System.exit(1); } - private static void tryCreateBucket() throws IOException { - View.header1("Trying to create a new bucket " + settings.getBucket()); - Storage.Buckets.Insert insertBucket = storage.buckets() - .insert(settings.getProject(), new Bucket().setName(settings.getBucket()).setLocation("US") - // .setDefaultObjectAcl(ImmutableList.of( - // new ObjectAccessControl().setEntity("allAuthenticatedUsers").setRole("READER"))) - ); - try { - @SuppressWarnings("unused") - Bucket createdBucket = insertBucket.execute(); - } catch (GoogleJsonResponseException e) { - GoogleJsonError error = e.getDetails(); - if (error.getCode() == HTTP_CONFLICT - && error.getMessage().contains("You already own this bucket.")) { - System.out.println("already exists"); - } else { - throw e; - } - } - } - - private static void getBucket() throws IOException { - View.header1("Getting bucket " + settings.getBucket() + " metadata"); - Storage.Buckets.Get getBucket = storage.buckets().get(settings.getBucket()); - getBucket.setProjection("full"); - Bucket bucket = getBucket.execute(); - View.show(bucket); - } - - private static void listObjects() throws IOException { - View.header1("Listing objects in bucket " + settings.getBucket()); - Storage.Objects.List listObjects = storage.objects().list(settings.getBucket()); - listObjects.setMaxResults(5L); - Objects objects = listObjects.execute(); - // Keep track of the page number in case we're listing objects - // for a bucket with thousands of objects. We'll limit ourselves - // to 5 pages - int currentPageNumber = 0; - while (objects.getItems() != null && !objects.getItems().isEmpty() - && ++currentPageNumber <= 5) { - for (StorageObject object : objects.getItems()) { - View.show(object); - View.separator(); - } - // Fetch the next page - String nextPageToken = objects.getNextPageToken(); - if (nextPageToken == null) { - break; - } - listObjects.setPageToken(nextPageToken); - View.header2("New page of objects"); - objects = listObjects.execute(); - } - } - - private static void getObjectMetadata() throws IOException { - View.header1("Getting object metadata from gs://pub/SomeOfTheTeam.jpg"); - Storage.Objects.Get getObject = storage.objects().get("pub", "SomeOfTheTeam.jpg"); - StorageObject object = getObject.execute(); - View.show(object); - } - - /** - * Generates a random data block and repeats it to provide the stream. - * - * Using a buffer instead of just filling from java.util.Random because the latter causes - * noticeable lag in stream reading, which detracts from upload speed. This class takes all that - * cost in the constructor. - */ - private static class RandomDataBlockInputStream extends InputStream { - - private long byteCountRemaining; - private final byte[] buffer; - - public RandomDataBlockInputStream(long size, int blockSize) { - byteCountRemaining = size; - final Random random = new Random(); - buffer = new byte[blockSize]; - random.nextBytes(buffer); - } - - /* - * (non-Javadoc) - * - * @see java.io.InputStream#read() - */ - @Override - public int read() { - throw new AssertionError("Not implemented; too slow."); - } - - /* - * (non-Javadoc) - * - * @see java.io.InputStream#read(byte [], int, int) - */ - @Override - public int read(byte b[], int off, int len) { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return 0; - } else if (byteCountRemaining == 0) { - return -1; - } - int actualLen = len > byteCountRemaining ? (int) byteCountRemaining : len; - for (int i = off; i < actualLen; i++) { - b[i] = buffer[i % buffer.length]; - } - byteCountRemaining -= actualLen; - return actualLen; - } - } - - private static class CustomUploadProgressListener implements MediaHttpUploaderProgressListener { - private final Stopwatch stopwatch = new Stopwatch(); - - public CustomUploadProgressListener() { - } - - @Override - public void progressChanged(MediaHttpUploader uploader) { - switch (uploader.getUploadState()) { - case INITIATION_STARTED: - stopwatch.start(); - System.out.println("Initiation has started!"); - break; - case INITIATION_COMPLETE: - System.out.println("Initiation is complete!"); - break; - case MEDIA_IN_PROGRESS: - // TODO(nherring): Progress works iff you have a content length specified. - // System.out.println(uploader.getProgress()); - System.out.println(uploader.getNumBytesUploaded()); - break; - case MEDIA_COMPLETE: - stopwatch.stop(); - System.out.println(String.format("Upload is complete! (%s)", stopwatch)); - break; - case NOT_STARTED: - break; - } - } - } - - private static void uploadObject(boolean useCustomMetadata) throws IOException { - View.header1("Uploading object."); - final long objectSize = 100 * 1000 * 1000 /* 100 MB */; - InputStreamContent mediaContent = new InputStreamContent( - "application/octet-stream", new RandomDataBlockInputStream(objectSize, 1024)); - // Not strictly necessary, but allows optimization in the cloud. - // mediaContent.setLength(OBJECT_SIZE); - - StorageObject objectMetadata = null; - - if (useCustomMetadata) { - // If you have custom settings for metadata on the object you want to set - // then you can allocate a StorageObject and set the values here. You can - // leave out setBucket(), since the bucket is in the insert command's - // parameters. - List acl = Lists.newArrayList(); - if (settings.getEmail() != null && !settings.getEmail().isEmpty()) { - acl.add( - new ObjectAccessControl().setEntity("user-" + settings.getEmail()).setRole("OWNER")); - } - if (settings.getDomain() != null && !settings.getDomain().isEmpty()) { - acl.add(new ObjectAccessControl().setEntity("domain-" + settings.getDomain()) - .setRole("READER")); - } - objectMetadata = new StorageObject().setName(settings.getPrefix() + "myobject") - .setMetadata(ImmutableMap.of("key1", "value1", "key2", "value2")).setAcl(acl) - .setContentDisposition("attachment"); - } - - Storage.Objects.Insert insertObject = - storage.objects().insert(settings.getBucket(), objectMetadata, mediaContent); - - if (!useCustomMetadata) { - // If you don't provide metadata, you will have specify the object - // name by parameter. You will probably also want to ensure that your - // default object ACLs (a bucket property) are set appropriately: - // https://developers.google.com/storage/docs/json_api/v1/buckets#defaultObjectAcl - insertObject.setName(settings.getPrefix() + "myobject"); - } - - insertObject.getMediaHttpUploader() - .setProgressListener(new CustomUploadProgressListener()).setDisableGZipContent(true); - // For small files, you may wish to call setDirectUploadEnabled(true), to - // reduce the number of HTTP requests made to the server. - if (mediaContent.getLength() > 0 && mediaContent.getLength() <= 2 * 1000 * 1000 /* 2MB */) { - insertObject.getMediaHttpUploader().setDirectUploadEnabled(true); - } - insertObject.execute(); - } - - private static class CustomDownloadProgressListener implements MediaHttpDownloaderProgressListener { - private final Stopwatch stopwatch; - - public CustomDownloadProgressListener(final Stopwatch stopwatch) { - this.stopwatch = stopwatch; - } - - @Override - public void progressChanged(MediaHttpDownloader downloader) { - switch (downloader.getDownloadState()) { - case MEDIA_IN_PROGRESS: - System.out.println(downloader.getProgress()); - break; - case MEDIA_COMPLETE: - stopwatch.stop(); - System.out.println(String.format("Download is complete! (%s)", stopwatch)); - break; - case NOT_STARTED: - break; - } - } - } - - private static void getObjectData() throws IOException { - View.header1("Getting object data uploaded object."); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Storage.Objects.Get getObject = - storage.objects().get(settings.getBucket(), settings.getPrefix() + "myobject"); - - Stopwatch stopwatch = new Stopwatch(); - getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE) - .setProgressListener(new CustomDownloadProgressListener(stopwatch.start())); - getObject.executeMediaAndDownloadTo(out); - System.out.println(new String(Arrays.copyOfRange(out.toByteArray(), 0, 5 * 80))); - if (out.size() != 5 * 80) { - System.out.println("...truncated..."); - } - System.out.println(String.format("Output buffer was size %d.", out.size())); - } - - private static void getPartialObjectData() throws IOException { - View.header1("Getting part of object data uploaded object."); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - Storage.Objects.Get getObject = - storage.objects().get(settings.getBucket(), settings.getPrefix() + "myobject"); - getObject.setRequestHeaders(new HttpHeaders().setRange( - String.format("bytes=%d-%d", 2 * 1000 * 1000, 3 * 1000 * 1000 - 1))); // 2-3MB - - Stopwatch stopwatch = new Stopwatch(); - getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE) - .setProgressListener(new CustomDownloadProgressListener(stopwatch.start())); - getObject.executeMediaAndDownloadTo(out); - System.out.println(new String(Arrays.copyOfRange(out.toByteArray(), 0, 5 * 80))); - if (out.size() != 5 * 80) { - System.out.println("...truncated..."); - } - System.out.println(String.format("Output buffer was size %d.", out.size())); - } - - // TODO(nherring): get a crc32 implementation, e.g., http://goo.gl/4oOlY } diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/View.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/View.java new file mode 100644 index 00000000..c2887554 --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/View.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.cmdline; + +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.StorageObject; + +/** Provides simple cmdline UI. */ +public class View { + + static void header1(String name) { + System.out.println(); + System.out.println("================== " + name + " =================="); + System.out.println(); + } + + static void header2(String name) { + System.out.println(); + System.out.println("~~~~~~~~~~~~~~~~~~ " + name + " ~~~~~~~~~~~~~~~~~~"); + System.out.println(); + } + + static void show(Bucket bucket) { + System.out.println("name: " + bucket.getName()); + System.out.println("location: " + bucket.getLocation()); + System.out.println("timeCreated: " + bucket.getTimeCreated()); + System.out.println("owner: " + bucket.getOwner()); + System.out.println("acl: " + bucket.getAcl()); + } + + static void show(StorageObject object) { + System.out.println("name: " + object.getName()); + System.out.println("size: " + object.getSize()); + System.out.println("contentType: " + object.getContentType()); + System.out.println("updated: " + object.getUpdated()); + System.out.println("owner: " + object.getOwner()); + // should only show up if projection is full. + // System.out.println("acl: " + object.getAcl()); + } + + static void separator() { + System.out.println(); + System.out.println("------------------------------------------------------"); + System.out.println(); + } +} + diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsGetExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsGetExample.java new file mode 100644 index 00000000..a2381ecf --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsGetExample.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.examples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.samples.storage.util.CredentialsProvider; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.Bucket; + +import java.io.IOException; + + +/** Example of retrieving a GCS bucket's metadata. */ +public class BucketsGetExample { + + private static final String BUCKET_NAME = "*** bucket name ***"; + + public static Bucket get(Storage storage, String bucketName) throws IOException { + Storage.Buckets.Get getBucket = storage.buckets().get(bucketName); + getBucket.setProjection("full"); // if you are interested in acls. + return getBucket.execute(); + } + + public static void main(String[] args) throws Exception { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("Google-BucketsGetExample/1.0").build(); + get(storage, BUCKET_NAME); + } + +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java new file mode 100644 index 00000000..924b9a64 --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.examples; + +import static java.net.HttpURLConnection.HTTP_CONFLICT; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.json.GoogleJsonError; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.samples.storage.util.CredentialsProvider; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.Bucket; + +import java.io.IOException; + + +/** Example of creating a GCS bucket. */ +public class BucketsInsertExample { + + private static final String PROJECT_NAME = "*** project name ***"; + private static final String BUCKET_NAME = "*** bucket name ***"; + private static final String BUCKET_LOCATION = "*** bucket location ***"; + + public static Bucket insert(Storage storage, Bucket bucket) throws IOException { + return insertInNamedProject(storage, bucket.getProjectNumber().toString(), bucket); + } + + public static Bucket insertInNamedProject(Storage storage, String project, Bucket bucket) throws IOException { + try { + Storage.Buckets.Insert insertBucket = storage.buckets().insert(project, bucket); + return insertBucket.execute(); + } catch (GoogleJsonResponseException e) { + GoogleJsonError error = e.getDetails(); + if (error.getCode() == HTTP_CONFLICT + && error.getMessage().contains("You already own this bucket.")) { + System.out.println("already exists"); + return bucket; + } + System.err.println(error.getMessage()); + throw e; + } + } + + public static void main(String[] args) throws Exception { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("Google-BucketsInsertExample/1.0").build(); + insertInNamedProject(storage, PROJECT_NAME, new Bucket().setName(BUCKET_NAME).setLocation(BUCKET_LOCATION)); + } + +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java new file mode 100644 index 00000000..b66e860a --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.examples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.samples.storage.util.CredentialsProvider; +import com.google.api.services.storage.Storage; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public class ObjectsDownloadExample { + + private static final boolean IS_APP_ENGINE = false; + + private static final String BUCKET_NAME = "*** bucket name ***"; + private static final String OBJECT_NAME = "*** object name ***"; + + public static InputStream download(Storage storage, String bucketName, String objectName) + throws IOException { + Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); + getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE); + return getObject.executeMediaAsInputStream(); + } + + public static void downloadToOutputStream(Storage storage, String bucketName, String objectName, OutputStream data) + throws IOException { + Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); + getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE); + getObject.executeMediaAndDownloadTo(data); + } + + public static void downloadRangeToOutputStream(Storage storage, String bucketName, String objectName, + long firstBytePos, long lastBytePos, OutputStream data) throws IOException { + Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); + // Remove cast after https://github.com/google/google-api-java-client/issues/937 is addressed. + getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE) + .setContentRange(firstBytePos, (int) lastBytePos); + getObject.executeMediaAndDownloadTo(data); + } + + public static void main(String[] args) throws Exception { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("Google-ObjectsDownloadExample/1.0").build(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + downloadToOutputStream(storage, BUCKET_NAME, OBJECT_NAME, out); + System.out.println("Downloaded " + out.toByteArray().length + " bytes"); + } +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java new file mode 100644 index 00000000..72dcfb90 --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.examples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.samples.storage.util.CredentialsProvider; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.StorageObject; + +import java.io.IOException; + + +/** Example of getting GCS object metadata. */ +public class ObjectsGetExample { + + private static final String BUCKET_NAME = "*** bucket name ***"; + private static final String OBJECT_NAME = "*** object name ***"; + + public static StorageObject get(Storage storage, String bucketName, String objectName) throws IOException { + Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); + return getObject.execute(); + } + + public static void main(String[] args) throws Exception { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("Google-ObjectsGetExample/1.0").build(); + StorageObject object = get(storage, BUCKET_NAME, OBJECT_NAME); + System.out.println(object.getName() + " (size: " + object.getSize() + ")"); + } + +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java new file mode 100644 index 00000000..02fc4d7a --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.examples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.samples.storage.util.CredentialsProvider; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.Objects; +import com.google.api.services.storage.model.StorageObject; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +import java.io.IOException; +import java.util.List; + +public class ObjectsListExample { + + private static final String BUCKET_NAME = "*** bucket name ***"; + + public static Iterable list(Storage storage, String bucketName) throws IOException { + List> pagedList = Lists.newArrayList(); + Storage.Objects.List listObjects = storage.objects().list(bucketName); + Objects objects; + do { + objects = listObjects.execute(); + pagedList.add(objects.getItems()); + listObjects.setPageToken(objects.getNextPageToken()); + } while (objects.getNextPageToken() != null); + return Iterables.concat(pagedList); + } + + public static void main(String[] args) throws Exception { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("Google-ObjectsListExample/1.0").build(); + for (StorageObject object : list(storage, BUCKET_NAME)) { + System.out.println(object.getName() + " (size: " + object.getSize() + ")"); + } + } +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java new file mode 100644 index 00000000..28ae2496 --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.examples; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.http.InputStreamContent; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.samples.storage.util.CredentialsProvider; +import com.google.api.services.storage.Storage; +import com.google.api.services.storage.model.StorageObject; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; + +public class ObjectsUploadExample { + + private static final String BUCKET_NAME = "*** bucket name ***"; + private static final String OBJECT_NAME = "*** object name ***"; + private static final String FILE_NAME = "*** upload file name ***"; + + public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, String data) + throws UnsupportedEncodingException, IOException { + return uploadSimple(storage, bucketName, objectName, new ByteArrayInputStream(data.getBytes("UTF-8")), + "text/plain"); + } + + public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, File data) + throws FileNotFoundException, IOException { + return uploadSimple(storage, bucketName, objectName, new FileInputStream(data), "application/octet-stream"); + } + + public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, InputStream data, + String contentType) throws IOException { + InputStreamContent mediaContent = new InputStreamContent(contentType, data); + Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, null, mediaContent).setName(objectName); + insertObject.getMediaHttpUploader().setDisableGZipContent(true); + return insertObject.execute(); + } + + public static StorageObject uploadWithMetadata(Storage storage, StorageObject object, InputStream data) + throws IOException { + InputStreamContent mediaContent = new InputStreamContent(object.getContentType(), data); + Storage.Objects.Insert insertObject = storage.objects().insert(object.getBucket(), object, mediaContent); + insertObject.getMediaHttpUploader().setDisableGZipContent(true); + return insertObject.execute(); + } + + public static void main(String[] args) throws Exception { + HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) + .setApplicationName("Google-ObjectsGetExample/1.0").build(); + StorageObject object = uploadSimple(storage, BUCKET_NAME, OBJECT_NAME, new File(FILE_NAME)); + System.out.println(object.getName() + " (size: " + object.getSize() + ")"); + } + +} diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java new file mode 100644 index 00000000..6d37fa4f --- /dev/null +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.api.services.samples.storage.util; + +import com.google.api.client.auth.oauth2.Credential; +import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; +import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; +import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; +import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.HttpTransport; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.util.store.DataStoreFactory; +import com.google.api.client.util.store.FileDataStoreFactory; +import com.google.api.services.samples.storage.cmdline.StorageSample; +import com.google.api.services.storage.StorageScopes; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Collections; + +/** + * Utility class to provide credentials and cache them in a local file. + */ +public class CredentialsProvider { + + /** Directory to store user credentials. */ + private static final java.io.File DATA_STORE_DIR = + new java.io.File(System.getProperty("user.home"), ".store/storage_sample"); + + /** + * Global instance of the {@link DataStoreFactory}. The best practice is to make it a single + * globally shared instance across your application, but for this sample, it's only useful + * for saving credentials across runs. + */ + private static FileDataStoreFactory dataStoreFactory; + + /** + * Authorizes the installed application to access user's protected data. + * + *

If you plan to run on AppEngine or Compute Engine, consider instead + * {@link GoogleCredential#getApplicationDefault()}, which will use the ambient credentials + * for the project's service-account. + */ + public static Credential authorize(HttpTransport httpTransport, JsonFactory jsonFactory) + throws IOException { + dataStoreFactory = new FileDataStoreFactory(DATA_STORE_DIR); + // load client secrets + GoogleClientSecrets clientSecrets = GoogleClientSecrets.load(jsonFactory, + new InputStreamReader(StorageSample.class.getResourceAsStream("/client_secrets.json"))); + if (clientSecrets.getDetails().getClientId().startsWith("Enter") + || clientSecrets.getDetails().getClientSecret().startsWith("Enter ")) { + System.out.println( + "Enter Client ID and Secret from https://console.developers.google.com/project/_/apiui/credential " + + "into storage-cmdline-sample/src/main/resources/client_secrets.json"); + System.exit(1); + } + // set up authorization code flow + GoogleAuthorizationCodeFlow flow = new GoogleAuthorizationCodeFlow.Builder( + httpTransport, jsonFactory, clientSecrets, + Collections.singleton(StorageScopes.DEVSTORAGE_FULL_CONTROL)).setDataStoreFactory( + dataStoreFactory).build(); + // authorize + return new AuthorizationCodeInstalledApp(flow, new LocalServerReceiver()).authorize("user"); + } + +} From 379138f79783c951b45e489758727d5dae00a577 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Fri, 1 May 2015 16:40:56 -0700 Subject: [PATCH 03/10] Remove no-longer-referenced settings --- .../src/main/resources/sample_settings.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/storage-cmdline-sample/src/main/resources/sample_settings.json b/storage-cmdline-sample/src/main/resources/sample_settings.json index 4730becb..da64cbda 100644 --- a/storage-cmdline-sample/src/main/resources/sample_settings.json +++ b/storage-cmdline-sample/src/main/resources/sample_settings.json @@ -1,7 +1,5 @@ { "project": "Enter Project", "bucket": "Enter a bucket name to create", - "prefix": "Enter an object name prefix for objects created by the sample; can leave blank", - "email": "Enter an email that has a Google account; leave blank to skip", - "domain": "Enter a Google Apps domain; leave blank to skip" + "prefix": "Enter an object name prefix for objects created by the sample; can leave blank" } From 37b57fdd4869272213494e57795b1779a47583c4 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Mon, 4 May 2015 16:31:31 -0700 Subject: [PATCH 04/10] Add crc32c calculation/validation. --- .../samples/storage/cmdline/StorageSample.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java index 35676c08..4c507594 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java @@ -33,11 +33,15 @@ import com.google.api.services.storage.model.Bucket; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableMap; +import com.google.common.hash.Hashing; +import com.google.common.hash.HashingOutputStream; import com.google.common.io.BaseEncoding; +import com.google.common.primitives.Ints; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.security.DigestOutputStream; import java.security.MessageDigest; @@ -92,6 +96,8 @@ public static void main(String[] args) { object = ObjectsUploadExample.uploadWithMetadata(storage, object, data); View.show(object); System.out.println("md5Hash: " + object.getMd5Hash()); + System.out.println("crc32c: " + object.getCrc32c() + ", decoded to " + + ByteBuffer.wrap(BaseEncoding.base64().decode(object.getCrc32c())).getInt()); View.header1("Getting object data of uploaded object, calculate hashes/crcs."); OutputStream nullOutputStream = new OutputStream() { @@ -101,11 +107,19 @@ public static void main(String[] args) { }; DigestOutputStream md5DigestOutputStream = new DigestOutputStream( nullOutputStream, MessageDigest.getInstance("MD5")); + HashingOutputStream crc32cHashingOutputStream = new HashingOutputStream(Hashing.crc32c(), md5DigestOutputStream); ObjectsDownloadExample.downloadToOutputStream( - storage, settings.getBucket(), settings.getPrefix() + "myobject", md5DigestOutputStream); + storage, settings.getBucket(), settings.getPrefix() + "myobject", crc32cHashingOutputStream); String calculatedMD5 = BaseEncoding.base64().encode(md5DigestOutputStream.getMessageDigest().digest()); System.out.println("md5Hash: " + calculatedMD5 + " " + (object.getMd5Hash().equals(calculatedMD5) ? "(MATCHES)" : "(MISMATCHES; data altered in transit)")); + int calculatedCrc32c = crc32cHashingOutputStream.hash().asInt(); + String calculatedEncodedCrc32c = BaseEncoding.base64().encode(Ints.toByteArray(calculatedCrc32c)); + // NOTE: Don't compare HashCode.asBytes() directly, as that encodes the crc32c in little-endien. + // You would have to reverseBytes first. + System.out.println("crc32c: " + calculatedEncodedCrc32c + ", decoded to " + + crc32cHashingOutputStream.hash().asInt() + " " + (object.getCrc32c().equals(calculatedEncodedCrc32c) + ? "(MATCHES)" : "(MISMATCHES; data altered in transit)")); // success! return; From 7df31487d38cac04a0061e66b22d3f22293072ce Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Mon, 4 May 2015 16:43:22 -0700 Subject: [PATCH 05/10] Fix guava dependency. Since API client includes Guava, its version obscures the explicit dependency. --- storage-cmdline-sample/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/storage-cmdline-sample/pom.xml b/storage-cmdline-sample/pom.xml index dbe3e250..256ec4ee 100644 --- a/storage-cmdline-sample/pom.xml +++ b/storage-cmdline-sample/pom.xml @@ -80,16 +80,16 @@ - - com.google.api-client - google-api-client - ${project.api.version} - com.google.guava guava 18.0 + + com.google.api-client + google-api-client + ${project.api.version} + com.google.oauth-client google-oauth-client-jetty From 9082525d6d7e401b97c096ff7efe319bfcf497a0 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Mon, 4 May 2015 16:46:48 -0700 Subject: [PATCH 06/10] Add sample-specific .gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don’t track Eclipse-generated directories. --- storage-cmdline-sample/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 storage-cmdline-sample/.gitignore diff --git a/storage-cmdline-sample/.gitignore b/storage-cmdline-sample/.gitignore new file mode 100644 index 00000000..e420ee4b --- /dev/null +++ b/storage-cmdline-sample/.gitignore @@ -0,0 +1 @@ +target/* From e5ff712b23fddfb0d5ae32be71e847c3a22530c4 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Tue, 5 May 2015 07:56:58 -0700 Subject: [PATCH 07/10] Applied code review commentary From comments on https://codereview.appspot.com/236910043/ patch set 1. --- .../services/samples/storage/cmdline/Helpers.java | 2 +- .../samples/storage/cmdline/StorageSample.java | 14 +++++++------- .../storage/examples/BucketsInsertExample.java | 10 +++++----- .../storage/examples/ObjectsDownloadExample.java | 5 +++++ ...Example.java => ObjectsGetMetadataExample.java} | 2 +- .../storage/examples/ObjectsUploadExample.java | 3 +++ 6 files changed, 22 insertions(+), 14 deletions(-) rename storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/{ObjectsGetExample.java => ObjectsGetMetadataExample.java} (98%) diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java index 1aaa1af1..a3d28e58 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java @@ -22,7 +22,7 @@ public class Helpers { /** * Generates a random data block and repeats it to provide the stream. * - * Using a buffer instead of just filling from java.util.Random because the latter causes + *

Using a buffer instead of just filling from java.util.Random because the latter causes * noticeable lag in stream reading, which detracts from upload speed. This class takes all that * cost in the constructor. */ diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java index 35676c08..804d761f 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java @@ -25,7 +25,7 @@ import com.google.api.services.samples.storage.examples.BucketsGetExample; import com.google.api.services.samples.storage.examples.BucketsInsertExample; import com.google.api.services.samples.storage.examples.ObjectsDownloadExample; -import com.google.api.services.samples.storage.examples.ObjectsGetExample; +import com.google.api.services.samples.storage.examples.ObjectsGetMetadataExample; import com.google.api.services.samples.storage.examples.ObjectsListExample; import com.google.api.services.samples.storage.examples.ObjectsUploadExample; import com.google.api.services.samples.storage.util.CredentialsProvider; @@ -54,17 +54,17 @@ public static void main(String[] args) { try { // initialize network, sample settings, credentials, and the storage client. HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); - JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); - SampleSettings settings = SampleSettings.load(JSON_FACTORY); - Credential credential = CredentialsProvider.authorize(httpTransport, JSON_FACTORY); - Storage storage = new Storage.Builder(httpTransport, JSON_FACTORY, credential) + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + SampleSettings settings = SampleSettings.load(jsonFactory); + Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); + Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) .setApplicationName(APPLICATION_NAME).build(); // // run commands // View.header1("Trying to create a new bucket " + settings.getBucket()); - BucketsInsertExample.insertInNamedProject(storage, settings.getProject(), + BucketsInsertExample.createInProject(storage, settings.getProject(), new Bucket().setName(settings.getBucket()).setLocation("US")); View.header1("Getting bucket " + settings.getBucket() + " metadata"); @@ -77,7 +77,7 @@ public static void main(String[] args) { } View.header1("Getting object metadata from gs://pub/SomeOfTheTeam.jpg"); - StorageObject object = ObjectsGetExample.get(storage, "pub", "SomeOfTheTeam.jpg"); + StorageObject object = ObjectsGetMetadataExample.get(storage, "pub", "SomeOfTheTeam.jpg"); View.show(object); View.header1("Uploading object."); diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java index 924b9a64..383db1c5 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java @@ -37,17 +37,17 @@ public class BucketsInsertExample { private static final String BUCKET_NAME = "*** bucket name ***"; private static final String BUCKET_LOCATION = "*** bucket location ***"; - public static Bucket insert(Storage storage, Bucket bucket) throws IOException { - return insertInNamedProject(storage, bucket.getProjectNumber().toString(), bucket); + public static Bucket create(Storage storage, Bucket bucket) throws IOException { + return createInProject(storage, bucket.getProjectNumber().toString(), bucket); } - public static Bucket insertInNamedProject(Storage storage, String project, Bucket bucket) throws IOException { + public static Bucket createInProject(Storage storage, String project, Bucket bucket) throws IOException { try { Storage.Buckets.Insert insertBucket = storage.buckets().insert(project, bucket); return insertBucket.execute(); } catch (GoogleJsonResponseException e) { GoogleJsonError error = e.getDetails(); - if (error.getCode() == HTTP_CONFLICT + if (error != null && error.getCode() == HTTP_CONFLICT && error.getMessage().contains("You already own this bucket.")) { System.out.println("already exists"); return bucket; @@ -63,7 +63,7 @@ public static void main(String[] args) throws Exception { Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) .setApplicationName("Google-BucketsInsertExample/1.0").build(); - insertInNamedProject(storage, PROJECT_NAME, new Bucket().setName(BUCKET_NAME).setLocation(BUCKET_LOCATION)); + createInProject(storage, PROJECT_NAME, new Bucket().setName(BUCKET_NAME).setLocation(BUCKET_LOCATION)); } } diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java index b66e860a..fdb0da7d 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java @@ -48,6 +48,11 @@ public static void downloadToOutputStream(Storage storage, String bucketName, St getObject.executeMediaAndDownloadTo(data); } + /** + * This shows how to download a portion of an object. Especially useful for + * resuming after a download fails, but can also be used to download in + * parallel. + */ public static void downloadRangeToOutputStream(Storage storage, String bucketName, String objectName, long firstBytePos, long lastBytePos, OutputStream data) throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java similarity index 98% rename from storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java rename to storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java index 72dcfb90..82dc79df 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java @@ -27,7 +27,7 @@ /** Example of getting GCS object metadata. */ -public class ObjectsGetExample { +public class ObjectsGetMetadataExample { private static final String BUCKET_NAME = "*** bucket name ***"; private static final String OBJECT_NAME = "*** object name ***"; diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java index 28ae2496..9da58f1b 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java @@ -53,6 +53,9 @@ public static StorageObject uploadSimple(Storage storage, String bucketName, Str String contentType) throws IOException { InputStreamContent mediaContent = new InputStreamContent(contentType, data); Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, null, mediaContent).setName(objectName); + // The media uploader gzips content by default, and alters the Content-Encoding accordingly. + // GCS dutifully stores content as-uploaded. This line disables the media uploader behavior, + // so the service stores exactly what is in the InputStream, without transformation. insertObject.getMediaHttpUploader().setDisableGZipContent(true); return insertObject.execute(); } From 2135d2db266b51572f61583c9f5ecdceeb98cb33 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Tue, 5 May 2015 11:55:40 -0700 Subject: [PATCH 08/10] Checkstyle fixes Conform to 100 character line lengths and add JavaDoc to top-level classes. While here, also fixed the Application names of two of the examples to match the class name. --- .../samples/storage/cmdline/Helpers.java | 4 +++ .../storage/cmdline/StorageSample.java | 26 ++++++++++----- .../examples/BucketsInsertExample.java | 6 ++-- .../examples/ObjectsDownloadExample.java | 11 ++++--- .../examples/ObjectsGetMetadataExample.java | 5 +-- .../storage/examples/ObjectsListExample.java | 5 ++- .../examples/ObjectsUploadExample.java | 33 +++++++++++-------- .../storage/util/CredentialsProvider.java | 4 +-- 8 files changed, 60 insertions(+), 34 deletions(-) diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java index a3d28e58..4e28d545 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/Helpers.java @@ -17,6 +17,10 @@ import java.io.InputStream; import java.util.Random; + +/** + * Support classes for the command-line sample. + */ public class Helpers { /** diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java index 17540d98..64e4065c 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/cmdline/StorageSample.java @@ -46,6 +46,9 @@ import java.security.MessageDigest; +/** + * Sample for Google Cloud Storage JSON API client. + */ public class StorageSample { /** @@ -107,18 +110,23 @@ public static void main(String[] args) { }; DigestOutputStream md5DigestOutputStream = new DigestOutputStream( nullOutputStream, MessageDigest.getInstance("MD5")); - HashingOutputStream crc32cHashingOutputStream = new HashingOutputStream(Hashing.crc32c(), md5DigestOutputStream); - ObjectsDownloadExample.downloadToOutputStream( - storage, settings.getBucket(), settings.getPrefix() + "myobject", crc32cHashingOutputStream); - String calculatedMD5 = BaseEncoding.base64().encode(md5DigestOutputStream.getMessageDigest().digest()); - System.out.println("md5Hash: " + calculatedMD5 + " " + (object.getMd5Hash().equals(calculatedMD5) + HashingOutputStream crc32cHashingOutputStream = new HashingOutputStream(Hashing.crc32c(), + md5DigestOutputStream); + ObjectsDownloadExample.downloadToOutputStream(storage, settings.getBucket(), + settings.getPrefix() + "myobject", crc32cHashingOutputStream); + String calculatedMD5 = BaseEncoding.base64().encode( + md5DigestOutputStream.getMessageDigest().digest()); + System.out.println("md5Hash: " + calculatedMD5 + " " + + (object.getMd5Hash().equals(calculatedMD5) ? "(MATCHES)" : "(MISMATCHES; data altered in transit)")); int calculatedCrc32c = crc32cHashingOutputStream.hash().asInt(); - String calculatedEncodedCrc32c = BaseEncoding.base64().encode(Ints.toByteArray(calculatedCrc32c)); - // NOTE: Don't compare HashCode.asBytes() directly, as that encodes the crc32c in little-endien. - // You would have to reverseBytes first. + String calculatedEncodedCrc32c = BaseEncoding.base64().encode( + Ints.toByteArray(calculatedCrc32c)); + // NOTE: Don't compare HashCode.asBytes() directly, as that encodes the crc32c in + // little-endien. One would have to reverse the bytes first. System.out.println("crc32c: " + calculatedEncodedCrc32c + ", decoded to " - + crc32cHashingOutputStream.hash().asInt() + " " + (object.getCrc32c().equals(calculatedEncodedCrc32c) + + crc32cHashingOutputStream.hash().asInt() + " " + + (object.getCrc32c().equals(calculatedEncodedCrc32c) ? "(MATCHES)" : "(MISMATCHES; data altered in transit)")); // success! diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java index 383db1c5..a45812c5 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/BucketsInsertExample.java @@ -41,7 +41,8 @@ public static Bucket create(Storage storage, Bucket bucket) throws IOException { return createInProject(storage, bucket.getProjectNumber().toString(), bucket); } - public static Bucket createInProject(Storage storage, String project, Bucket bucket) throws IOException { + public static Bucket createInProject(Storage storage, String project, Bucket bucket) + throws IOException { try { Storage.Buckets.Insert insertBucket = storage.buckets().insert(project, bucket); return insertBucket.execute(); @@ -63,7 +64,8 @@ public static void main(String[] args) throws Exception { Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) .setApplicationName("Google-BucketsInsertExample/1.0").build(); - createInProject(storage, PROJECT_NAME, new Bucket().setName(BUCKET_NAME).setLocation(BUCKET_LOCATION)); + createInProject(storage, PROJECT_NAME, new Bucket().setName(BUCKET_NAME) + .setLocation(BUCKET_LOCATION)); } } diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java index fdb0da7d..d70a6131 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsDownloadExample.java @@ -27,6 +27,8 @@ import java.io.InputStream; import java.io.OutputStream; + +/** Example of downloading a GCS object. */ public class ObjectsDownloadExample { private static final boolean IS_APP_ENGINE = false; @@ -41,8 +43,8 @@ public static InputStream download(Storage storage, String bucketName, String ob return getObject.executeMediaAsInputStream(); } - public static void downloadToOutputStream(Storage storage, String bucketName, String objectName, OutputStream data) - throws IOException { + public static void downloadToOutputStream(Storage storage, String bucketName, String objectName, + OutputStream data) throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE); getObject.executeMediaAndDownloadTo(data); @@ -53,8 +55,9 @@ public static void downloadToOutputStream(Storage storage, String bucketName, St * resuming after a download fails, but can also be used to download in * parallel. */ - public static void downloadRangeToOutputStream(Storage storage, String bucketName, String objectName, - long firstBytePos, long lastBytePos, OutputStream data) throws IOException { + public static void downloadRangeToOutputStream(Storage storage, String bucketName, + String objectName, long firstBytePos, long lastBytePos, OutputStream data) + throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); // Remove cast after https://github.com/google/google-api-java-client/issues/937 is addressed. getObject.getMediaHttpDownloader().setDirectDownloadEnabled(!IS_APP_ENGINE) diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java index 82dc79df..5f648cb9 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsGetMetadataExample.java @@ -32,7 +32,8 @@ public class ObjectsGetMetadataExample { private static final String BUCKET_NAME = "*** bucket name ***"; private static final String OBJECT_NAME = "*** object name ***"; - public static StorageObject get(Storage storage, String bucketName, String objectName) throws IOException { + public static StorageObject get(Storage storage, String bucketName, String objectName) + throws IOException { Storage.Objects.Get getObject = storage.objects().get(bucketName, objectName); return getObject.execute(); } @@ -42,7 +43,7 @@ public static void main(String[] args) throws Exception { JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) - .setApplicationName("Google-ObjectsGetExample/1.0").build(); + .setApplicationName("Google-ObjectsGetMetadataExample/1.0").build(); StorageObject object = get(storage, BUCKET_NAME, OBJECT_NAME); System.out.println(object.getName() + " (size: " + object.getSize() + ")"); } diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java index 02fc4d7a..c35e102d 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsListExample.java @@ -29,11 +29,14 @@ import java.io.IOException; import java.util.List; + +/** Example of listing objects in a GCS bucket. */ public class ObjectsListExample { private static final String BUCKET_NAME = "*** bucket name ***"; - public static Iterable list(Storage storage, String bucketName) throws IOException { + public static Iterable list(Storage storage, String bucketName) + throws IOException { List> pagedList = Lists.newArrayList(); Storage.Objects.List listObjects = storage.objects().list(bucketName); Objects objects; diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java index 9da58f1b..9102661a 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/examples/ObjectsUploadExample.java @@ -32,27 +32,31 @@ import java.io.InputStream; import java.io.UnsupportedEncodingException; + +/** Example of uploading data to create a GCS object. */ public class ObjectsUploadExample { private static final String BUCKET_NAME = "*** bucket name ***"; private static final String OBJECT_NAME = "*** object name ***"; private static final String FILE_NAME = "*** upload file name ***"; - public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, String data) - throws UnsupportedEncodingException, IOException { - return uploadSimple(storage, bucketName, objectName, new ByteArrayInputStream(data.getBytes("UTF-8")), - "text/plain"); + public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, + String data) throws UnsupportedEncodingException, IOException { + return uploadSimple(storage, bucketName, objectName, new ByteArrayInputStream( + data.getBytes("UTF-8")), "text/plain"); } - public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, File data) - throws FileNotFoundException, IOException { - return uploadSimple(storage, bucketName, objectName, new FileInputStream(data), "application/octet-stream"); + public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, + File data) throws FileNotFoundException, IOException { + return uploadSimple(storage, bucketName, objectName, new FileInputStream(data), + "application/octet-stream"); } - public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, InputStream data, - String contentType) throws IOException { + public static StorageObject uploadSimple(Storage storage, String bucketName, String objectName, + InputStream data, String contentType) throws IOException { InputStreamContent mediaContent = new InputStreamContent(contentType, data); - Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, null, mediaContent).setName(objectName); + Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, null, mediaContent) + .setName(objectName); // The media uploader gzips content by default, and alters the Content-Encoding accordingly. // GCS dutifully stores content as-uploaded. This line disables the media uploader behavior, // so the service stores exactly what is in the InputStream, without transformation. @@ -60,10 +64,11 @@ public static StorageObject uploadSimple(Storage storage, String bucketName, Str return insertObject.execute(); } - public static StorageObject uploadWithMetadata(Storage storage, StorageObject object, InputStream data) - throws IOException { + public static StorageObject uploadWithMetadata(Storage storage, StorageObject object, + InputStream data) throws IOException { InputStreamContent mediaContent = new InputStreamContent(object.getContentType(), data); - Storage.Objects.Insert insertObject = storage.objects().insert(object.getBucket(), object, mediaContent); + Storage.Objects.Insert insertObject = storage.objects().insert(object.getBucket(), object, + mediaContent); insertObject.getMediaHttpUploader().setDisableGZipContent(true); return insertObject.execute(); } @@ -73,7 +78,7 @@ public static void main(String[] args) throws Exception { JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); Credential credential = CredentialsProvider.authorize(httpTransport, jsonFactory); Storage storage = new Storage.Builder(httpTransport, jsonFactory, credential) - .setApplicationName("Google-ObjectsGetExample/1.0").build(); + .setApplicationName("Google-ObjectsUploadExample/1.0").build(); StorageObject object = uploadSimple(storage, BUCKET_NAME, OBJECT_NAME, new File(FILE_NAME)); System.out.println(object.getName() + " (size: " + object.getSize() + ")"); } diff --git a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java index 6d37fa4f..e0306609 100644 --- a/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java +++ b/storage-cmdline-sample/src/main/java/com/google/api/services/samples/storage/util/CredentialsProvider.java @@ -63,8 +63,8 @@ public static Credential authorize(HttpTransport httpTransport, JsonFactory json if (clientSecrets.getDetails().getClientId().startsWith("Enter") || clientSecrets.getDetails().getClientSecret().startsWith("Enter ")) { System.out.println( - "Enter Client ID and Secret from https://console.developers.google.com/project/_/apiui/credential " - + "into storage-cmdline-sample/src/main/resources/client_secrets.json"); + "Enter Client ID and Secret from https://console.developers.google.com/project/_/apiui/" + + "credential into storage-cmdline-sample/src/main/resources/client_secrets.json"); System.exit(1); } // set up authorization code flow From 1c1464a524eabd8c7fd785d1f8fa5938e8e4b2f2 Mon Sep 17 00:00:00 2001 From: Nathan Herring Date: Tue, 5 May 2015 12:26:11 -0700 Subject: [PATCH 09/10] Update instructions Change references from mercurial to git, and update the developer console URL to the modern use. --- storage-cmdline-sample/instructions.html | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/storage-cmdline-sample/instructions.html b/storage-cmdline-sample/instructions.html index 8451ff85..978eb127 100644 --- a/storage-cmdline-sample/instructions.html +++ b/storage-cmdline-sample/instructions.html @@ -8,15 +8,15 @@

Browse Online

Register Your Application

For more information about registering your application, see the Google Cloud Storage documentation - JSON Java Example. + JSON Java Example, + as setup for that simpler example is similar.

Checkout Instructions

@@ -45,7 +46,7 @@

Checkout Instructions

 cd [someDirectory]
-hg clone https://code.google.com/p/google-api-java-client.samples/ google-api-java-client-samples
+git clone https://github.com/google/google/google-api-java-client-samples.git
 cd google-api-java-client-samples/storage-cmdline-sample
 cp ~/Downloads/client_secrets.json src/main/resources/client_secrets.json
 # Edit the settings file and enter in appropriate values.

From 98f3c077d0fa1d904fbc8425f2961f5020a3f9fa Mon Sep 17 00:00:00 2001
From: Nathan Herring 
Date: Tue, 5 May 2015 13:44:37 -0700
Subject: [PATCH 10/10] Remove extra "/google" in instructions

---
 storage-cmdline-sample/instructions.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/storage-cmdline-sample/instructions.html b/storage-cmdline-sample/instructions.html
index 978eb127..f1159020 100644
--- a/storage-cmdline-sample/instructions.html
+++ b/storage-cmdline-sample/instructions.html
@@ -46,7 +46,7 @@ 
Checkout Instructions

 
   
 cd [someDirectory]
-git clone https://github.com/google/google/google-api-java-client-samples.git
+git clone https://github.com/google/google-api-java-client-samples.git
 cd google-api-java-client-samples/storage-cmdline-sample
 cp ~/Downloads/client_secrets.json src/main/resources/client_secrets.json
 # Edit the settings file and enter in appropriate values.