Pytorch

Pytorch 버전과 CUDA 버전 확인

import torch
print(f'Pytorch:{torch.__version__} (' + (f'CUDA {torch.version.cuda}' if torch.cuda.is_available() else "CPU") + ")")


torch.backends.cudnn.version()

import torch

torch_version = f'Pytorch:{torch.__version__}'

versions = []
if torch.cuda.is_available():
    cuda_version = f'CUDA {torch.version.cuda}'
    versions.append(cuda_version)
if torch.backends.cudnn.is_available():
    cudnn_version = 'cuDNN ' + ".".join(str(torch.backends.cudnn.version()))
    versions.append(cudnn_version)
if torch.backends.mkldnn.is_available():
    versions.append(f'mkldnn')
if torch.backends.openmp.is_available():
    versions.append('openmp')
print(torch_version)
print('(' + ",".join(versions) + ')')

List 나누기

def list_chunk(lst, n):
    return [lst[i:i+n] for i in range(0, len(lst), n)]

List Merge

def merge_list(lists: list):
    return [x for x in list(chain.from_iterable(zip_longest(*lists, fillvalue=None))) if x is not None]

url로 이미지 받아오기

def from_url(url: str):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
        request = urllib.request.Request(url, headers=headers)
        response = urllib.request.urlopen(request)
        image_data = response.read()
        image_array = np.asarray(bytearray(image_data), dtype=np.uint8)
        image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
        if image is None:  # gif일 경우
            gif = Image.open(io.BytesIO(image_data))
            gif.seek(0)
            image = np.array(gif.convert('RGB'))
        return image
    except Exception as e:
        print(e)
        return None

url encoded 된 문자열에서 공백이 +로 바뀌는 문제

text = urllib.parse.unquote(text_encoded.replace("+", "%20"))

단위 변환

def format_file_size(size_bytes: int, precision: int = 2, fmt: str | None = None, padding: str = ''):
    if size_bytes == 0: return "0B"
    import math
    units = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
    i = int(math.log(size_bytes, 1024)) if fmt is None else units.index(fmt)
    return f'{round(size_bytes / 1024 ** i, precision)}{padding}{units[i]}'



def ms_to_hms(milliseconds):
    hour = milliseconds // 3600000
    milliseconds %= 3600000
    minute = milliseconds // 60000
    milliseconds %= 60000
    second = milliseconds // 1000
    milliseconds %= 1000
    return hour, minute, second, milliseconds


def s_to_hms(seconds):
    seconds = seconds % (24 * 3600)
    hour = seconds // 3600
    seconds %= 3600
    minutes = seconds // 60
    seconds %= 60
    return hour, minutes, seconds

합치기

Dict 합치기

딕셔너리 합치기에는 대략 아래 5가지 방법이 있다.

그냥 무조건 update를 사용하면 된다.

import time
from typing import *


def merge_dict_chunks_01(dict_chunks: List[dict]):
    t_beg = time.time()
    qc_datas = {}
    for data in dict_chunks:
        qc_datas.update(data)
    print(f"01. update: {time.time() - t_beg:.2f}s, len: {len(qc_datas)}")


def merge_dict_chunks_02(dict_chunks: List[dict]):
    from functools import reduce
    import operator
    t_beg = time.time()
    qc_datas = reduce(operator.or_, dict_chunks)
    print(f"02. reduce: {time.time() - t_beg:.2f}s, len: {len(qc_datas)}")


def merge_dict_chunks_03(dict_chunks: List[dict]):
    t_beg = time.time()
    qc_datas = {k: v for d in dict_chunks for k, v in d.items()}
    print(f"03. for-for: {time.time() - t_beg:.2f}s, len: {len(qc_datas)}")


def merge_dict_chunks_04(dict_chunks: List[dict]):
    from collections import ChainMap
    t_beg = time.time()
    qc_datas = dict(ChainMap(*dict_chunks))
    print(f"04. ChainMap: {time.time() - t_beg:.2f}s, len: {len(qc_datas)}")


def merge_dict_chunks_05(dict_chunks: List[dict]):
    t_beg = time.time()
    qc_datas = {k: v for d in dict_chunks for k, v in d.items()}
    print(f"05. DictComprehension: {time.time() - t_beg:.2f}s, len: {len(qc_datas)}")

01. update: 51.36s, len: 131360530
02. reduce: 949.06s, len: 131360530
03. for-for: 63.63s, len: 131360530
04. ChainMap: 6706.53s, len: 131360530
05. DictComprehension: 65.21s, len: 131360530

URL

def get_url(host, params)
    return host + "?" + "&".join([f'{k}={v}' for k, v in params.items()])

Multi Processing

결과를 가져올 필요가 없는 경우 parmap이 제일 좋다.

parmap.map(func, params, pm_pbar=True, pm_parallel=True, pm_processes=os.cpu_count())

디버깅

현재 함수 이름 가져오기

print(sys._getframe().f_code.co_name)
print(inspect.currentframe().f_code.co_name)

pip ssl 에러

pip install --trusted-host pypi.python.org --trusted-host files.pythonhosted.org --trusted-host pypi.org <PACKAGE>

References

• https://velog.io/@boychaboy/python