S3 Chunk Upload

대용량 파일을 다루는 방법론 중 하나.

스토리지에 파일을 분할하여 전송한다.

"""
Serializer를 csv로 저장한다.
"""
model_queryset = Model.objects.filter(조건...)
my_serializer = Serializer(model_queryset, many=True)

ordered_fieldnames = OrderedDict(
	[
    	("csv 1번 열", None),
        ("csv 2번 열", None),
    ]
)

with open("csv파일.csv", "w", newline="") as csv_file:
	writer = csv.DictWriter(csv_file, fieldnames=ordered_fieldnames)
    writer.writeheader()

	for row in my_serializer.data:
		new_row = {
        	"csv 1번 열": "test1",
            "csv 2번 열": "test2"
		}
        writer.writerow(new_row)

with open("video_detail.csv", "rb") as csv_file:
	# 2. S3에 chunk upload
    print("S3 에 Chunk Upload 시작")
    try:
    	s3_config = Config(
        	region_name="ap-northeast-2",
	        signature_version="s3v4",
        )
        s3_client = boto3.client("s3", config=s3_config)
        bucket_name = AWS_STORAGE_BUCKET_NAME

		# UUID4 생성
        uuid_val = uuid.uuid4()

		# UUID4를 16진수(hex) 문자열로 변환
        hex_val = uuid_val.hex

		key_name = f"video_detail_{str(hex_val)}"
        key = f"static/csv/{key_name}.csv"
	
    	# chunk upload 옵션
		chunk_upload_config = TransferConfig(
        	multipart_threshold=1024 * 20,
            max_concurrency=5,
            multipart_chunksize=5 * 1024 * 1024,
            use_threads=True,
		)

		"""
        - Filename: 로컬 파일 명
        - Bucket: s3 버킷 이름
        - Key: S3 오브젝트 key 이름
        - ExtraArgs: you can set extra arguments in this param in a json string. You can refer this link for valid upload arguments.
        - Config: This is the TransferConfig object which we created in the previous step.
        """
        s3_client.upload_file(
        	Filename=csv_file.name,
            Bucket=bucket_name,
            Key=key,
            ExtraArgs=None,
            Config=chunk_upload_config,
		)
        print(f"S3 Chunk Upload 완료")
        file_url = f"https://{AWS_S3_CUSTOM_DOMAIN}/{key}"

except Exception as e:
	logger.error("S3 Chunk Upload Error")
    logger.error(e)
    raise Exception(str(e))