def read_chunks(filepath, chunk_size=1024*1024):
with open(filepath, 'rb') as f:
while chunk := f.read(chunk_size):
# 尝试解码,若截断则回退至最近完整字符
try:
yield chunk.decode('utf-8')
except UnicodeDecodeError as e:
valid_part = chunk[:e.start]
yield valid_part.decode('utf-8')
f.seek(-len(chunk[e.start:]), 1) # 回退
对于需随机访问的场景(如日志回溯),可建立偏移量索引。例如记录每行起始位置:
with open('access.log', 'r', encoding='utf-8') as f:
offsets = []
pos = 0
for line in f:
offsets.append(pos)
pos = f.tell()
# 随机读取第10000行
with open('access.log', 'r', encoding='utf-8') as f:
f.seek(offsets[9999])
line = f.readline()
在分布式系统中,可结合mmap实现内存映射文件操作。例如高效搜索日志内容:
import mmap
with open('big_data.txt', 'rb') as f:
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
if b'ERROR' in mm:
print('发现错误记录')
from concurrent.futures import ProcessPoolExecutor
def write_chunk(args):
filepath, data = args
with open(filepath, 'a') as f:
f.write(data)
with ProcessPoolExecutor() as executor:
executor.map(write_chunk, [('log1.txt', 'data1'), ('log2.txt', 'data2')])
对写入密集型场景,推荐使用队列缓冲后批量写入。例如日志系统:
import queue
import threading
log_queue = queue.Queue()
def writer_thread():
while True:
items = []
try:
while len(items) < 100: # 批量写入
items.append(log_queue.get(timeout=0.1))
except queue.Empty:
if not items: continue
with open('app.log', 'a') as f:
f.writelines(items)
threading.Thread(target=writer_thread, daemon=True).start()
# 其他线程只需:log_queue.put('log message\n')
import portalocker
with open('shared.log', 'a') as f:
portalocker.lock(f, portalocker.LOCK_EX)
f.write('exclusive write\n')
portalocker.unlock(f)
网友们都关心什么?
根据搜索数据分析,用户对以下话题高度关注,我们整理了深度解答:
《Python文件操作常见错误TOP5》
① 忘记关闭文件 → 用with替代手动close()
② 编码不匹配 → 显式指定encoding='utf-8'
③ Windows路径转义 → 用r'path\to\file'或正斜杠
④ 大文件内存溢出 → 用for line in f:逐行处理
⑤ 未处理BOM → 用encoding='utf-8-sig'
《CSV文件读写最佳实践》
使用csv模块而非手动分割:
import csv
with open('data.csv', 'w', newline='', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(['name', 'age'])
writer.writerow(['张三', 28])
注意:newline=''防止双倍换行,encoding='utf-8-sig'确保Excel兼容
《JSON与YAML配置文件处理》
JSON推荐json.tool格式化:
import json
with open('config.json') as f:
data = json.load(f)
with open('output.json', 'w') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
YAML需安装pyyaml,支持注释和多类型:
import yaml
with open('config.yaml') as f:
config = yaml.safe_load(f)
《文件监控与实时处理》
使用watchdog库监控目录变化:
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
class LogHandler(FileSystemEventHandler):
def on_modified(self, event):
if event.src_path.endswith('.log'):
process_log(event.src_path)
observer = Observer()
observer.schedule(LogHandler(), path='/var/log', recursive=False)
observer.start()
实战案例:日志分析系统核心模块
下面是一个生产级日志分析器的简化实现,整合了本文所有最佳实践:
import os
import gzip
import json
import logging
from pathlib import Path
from datetime import datetime
class LogAnalyzer:
def __init__(self, log_dir, output_dir):
self.log_dir = Path(log_dir).resolve()
self.output_dir = Path(output_dir).resolve()
self.output_dir.mkdir(parents=True, exist_ok=True)
self.setup_logging()
def setup_logging(self):
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(
self.output_dir / 'analysis.log',
encoding='utf-8-sig'
)
]
)
self.logger = logging.getLogger(__name__)
def analyze(self):
for log_file in self.log_dir.glob('*.log*'):
self.process_file(log_file)
def process_file(self, filepath):
try:
# 自动处理压缩文件
open_func = gzip.open if filepath.suffix == '.gz' else open
mode = 'rt' if filepath.suffix == '.gz' else 'r'
with open_func(filepath, mode, encoding='utf-8') as f:
error_count = 0
total_lines = 0
for line in f:
total_lines += 1
if 'ERROR' in line:
error_count += 1
self.log_error(filepath, line)
self.logger.info(
f'处理完成: {filepath.name} '
f'[总行数: {total_lines}, 错误: {error_count}]'
)
# 生成分析报告
report = {
'file': str(filepath),
'processed_at': datetime.now().isoformat(),
'stats': {
'total_lines': total_lines,
'errors': error_count
}
}
with open(
self.output_dir / f'{filepath.stem}_report.json',
'w',
encoding='utf-8'
) as f:
json.dump(report, f, indent=2, ensure_ascii=False)
except PermissionError:
self.logger.error(f'权限不足: {filepath}')
except UnicodeDecodeError as e:
self.logger.error(f'编码错误 {filepath}: {e}')
except Exception as e:
self.logger.critical(f'未预期错误 {filepath}: {e}')
def log_error(self, filepath, line):
with open(
self.output_dir / 'errors.txt',
'a',
encoding='utf-8-sig'
) as f:
f.write(f'{datetime.now().isoformat()} | {filepath.name} | {line}')
# 使用示例
if __name__ == '__main__':
analyzer = LogAnalyzer('/var/log/app', './output')
analyzer.analyze()