The following code will be written and used in Jupyter Notebook.
It is assumed that the environment has been built and set for GCP and GCS.
Running first.py
import os
import pandas as pd
from io import BytesIO
from pathlib import Path
from google.cloud import storage
from google.oauth2 import service_account
def get_project_credentials():
credentials = service_account.Credentials.from_service_account_file(
"xx/xx/xx/xx.json",#The path where the json file for the service account key is located
scopes=["https://www.googleapis.com/auth/cloud-platform"],
)
return credentials
def get_storage_client(credentials):
storage_client = storage.Client(
credentials = credentials,
project = credentials.project_id,
)
return storage_client
created_bucket.py
created_bucket_name = 'tmp_bucket_qitaqita'
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.create_bucket(created_bucket_name)
Note that the bucket name must be unique worldwide. (Initially I tried to create it with the name "tmp_bucket", but I got an error because it already exists.)
trans_to_bucket.py
bucket_name = 'tmp_bucket_qitaqita'
file_name = 'send_txt.txt' #Name of what to send
destination_blob_name = 'sended_txt.txt' #Name at the destination
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(destination_blob_name)
blob.upload_from_filename(file_name)
If you name the destination as follows, a directory called make_dir will be created and the files will be transferred to it.
destination_blob_name = 'make_dir/sended_txt_2.txt'
del_bucket_file.py
bucket_name = 'tmp_bucket_qitaqita'
file_name = 'sended_txt.txt'#The file name you want to delete in the bucket
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.get_bucket(bucket_name)
blob = bucket.blob(file_name)
blob.delete()
bucket_name.py
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
blobs = storage_client.list_buckets()
for blob in blobs:
print(blob.name)
filename_in_bucket.py
bucket_name = 'tmp_bucket_qitaqita'
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.get_bucket(bucket_name)
blobs = bucket.list_blobs()
for blob in blobs:
print(blob.name)
del_bucket.py
bucket_name = 'tmp_bucket_qitaqita'#Bucket name you want to delete
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.get_bucket(bucket_name)
bucket.delete()
Load & Save.py
bucket_name = 'tmp_bucket_qitaqita'
file_name = 'sended_txt.txt'#File name in the bucket you want to read
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.get_bucket(bucket_name)
blob = storage.Blob(file_name, bucket)
content = blob.download_as_string()
with open('sended_txt.txt', mode='wb') as f:
f.write(content)
Read csv file.py
bucket_name = 'tmp_bucket_qitaqita'
file_name = 'dataset_data.csv'
credentials = get_project_credentials()
storage_client = get_storage_client(credentials)
bucket = storage_client.get_bucket(bucket_name)
blob = storage.Blob(file_name, bucket)
content = blob.download_as_string()
df_csv = pd.read_csv(BytesIO(content))
Digression: This is the first post to qiita. I hope it helps.
Recommended Posts