Pythonでのjsonlの扱い方法

Pythonでjsonlを扱う際のPolars、Pandas、ライブラリ無しでの方法をメモします。

Polars
#

読み込み
#

python

import polars as pl

# Eager Evaluaiton
data_df = pl.read_ndjson("file.jsonl")

print(data_df.describe())

# Lazy Evaluaiton
data_df = pl.scan_ndjson("file.jsonl")

## Need to evaluation before describe() when lazy evaluation
data_df = data_df.fetch()
print(data_df.describe())

書き込み
#

python

import polars as pl

# sample data list[dict]
data_list = [{"name": "alice", "age": "18"},
             {"name": "bob", "age": "17"}]

data_df = pl.DataFrame(data_list)

data_df.write_ndjson("file.jsonl")

Pandas
#

読み込み
#

python

import pandas as pd

data_df = pd.read_json("file.jsonl", orient="records", lines=True)

print(data_df.info())

書き込み
#

python

import pandas as pd

# sample data list[dict]
data_list = [{"name": "alice", "age": "18"},
             {"name": "bob", "age": "17"}]

data_df = pd.DataFrame(data_list)

data_df.to_json("file.jsonl", orient="records", lines=True, force_ascii=False)

ライブラリ不使用
#

読み込み
#

python

import json

data_list = []

with open("file.jsonl", "r") as f:
    while True:
        line = f.readline()
        if not line:
            break
        data_list.append(json.loads(line))

print(data_list)

書き込み
#

python

import json

# sample data list[dict]
data_list = [{"name": "alice", "age": "18"},
             {"name": "bob", "age": "17"}]

with open("file.jsonl", mode="w") as f:
    for line in data_list:
        json.dump(line, f, ensure_ascii=False)
        f.write('\n')