案例：效能分析實戰
案例：效能分析實戰
本案例基於 .claude/lib/markdown_link_checker.py 的實際程式碼，展示如何用 cProfile 和 line_profiler 進行效能分析。
先備知識
- 模組四基礎章節
- Python 正則表達式基礎
問題背景
現有設計
markdown_link_checker.py 是一個 Markdown 連結檢查工具，核心功能是解析文件中的連結並驗證其有效性。以下是關鍵的解析方法：
1import re
2from typing import List, Dict
3
4class MarkdownLinkChecker:
5 """Markdown link checker with precompiled regex patterns"""
6
7 # Precompiled regex patterns at class level (good practice!)
8 INLINE_LINK_PATTERN = re.compile(
9 r'(?<!!)\[([^\]]+)\]\(([^)]+)\)'
10 )
11
12 REFERENCE_DEF_PATTERN = re.compile(
13 r'^\s*\[([^\]]+)\]:\s*(.+)$',
14 re.MULTILINE
15 )
16
17 REFERENCE_USE_PATTERN = re.compile(
18 r'\[([^\]]+)\]\[([^\]]+)\]'
19 )
20
21 def parse_markdown_links(self, content: str) -> List[Dict]:
22 """
23 Parse all links in Markdown content
24
25 Args:
26 content: Markdown content string
27
28 Returns:
29 list[dict]: List of links with text, target, line
30 """
31 links = []
32 lines = content.split('\n')
33
34 # First, collect reference-style link definitions
35 reference_defs = {}
36 for match in self.REFERENCE_DEF_PATTERN.finditer(content):
37 ref_name = match.group(1).lower()
38 ref_target = match.group(2).strip()
39 reference_defs[ref_name] = ref_target
40
41 # Track code block state
42 in_code_block = False
43
44 # Parse inline links line by line
45 for line_num, line in enumerate(lines, start=1):
46 # Check for code block boundaries
47 if line.strip().startswith("```"):
48 in_code_block = not in_code_block
49 continue
50
51 # Skip links inside code blocks
52 if in_code_block:
53 continue
54
55 # Inline links: [text](/python-advanced/04-cpython-internals/case-studies/profiling/target)
56 for match in self.INLINE_LINK_PATTERN.finditer(line):
57 links.append({
58 "text": match.group(1),
59 "target": match.group(2),
60 "line": line_num
61 })
62
63 # Reference-style links: [text][ref]
64 for match in self.REFERENCE_USE_PATTERN.finditer(line):
65 ref_name = match.group(2).lower()
66 if ref_name in reference_defs:
67 links.append({
68 "text": match.group(1),
69 "target": reference_defs[ref_name],
70 "line": line_num
71 })
72
73 return links效能問題
處理大型文件時可能出現：
- 正則表達式效率問題：複雜的 pattern 可能導致回溯
- 重複編譯正則表達式：若 pattern 在方法內定義，每次呼叫都會重新編譯
- 不必要的字串操作：
split()會建立新的字串列表 - 多次遍歷：分別處理引用定義和行內連結
進階解決方案
分析目標
- 找出效能瓶頸所在
- 量化各部分的時間消耗
- 驗證優化效果
實作步驟
步驟 1：使用 cProfile 進行函式級分析
cProfile 是 Python 標準庫的效能分析工具，可以測量每個函式的呼叫次數和執行時間。
1import cProfile
2import pstats
3from io import StringIO
4from pathlib import Path
5
6def profile_link_checker():
7 """Profile the markdown link checker with cProfile"""
8
9 # Create test content with many links
10 test_content = generate_test_content(num_links=1000)
11
12 checker = MarkdownLinkChecker()
13
14 # Create profiler
15 profiler = cProfile.Profile()
16
17 # Run profiling
18 profiler.enable()
19 for _ in range(100): # Run multiple times for better statistics
20 checker.parse_markdown_links(test_content)
21 profiler.disable()
22
23 # Analyze results
24 stream = StringIO()
25 stats = pstats.Stats(profiler, stream=stream)
26 stats.sort_stats('cumulative') # Sort by cumulative time
27 stats.print_stats(20) # Show top 20 functions
28
29 print(stream.getvalue())
30
31 return stats
32
33def generate_test_content(num_links: int) -> str:
34 """Generate test Markdown content with specified number of links"""
35 lines = ["# Test Document\n"]
36
37 for i in range(num_links):
38 if i % 3 == 0:
39 # Inline link
40 lines.append(f"Check out [Link {i}](https://example.com/{i})\n")
41 elif i % 3 == 1:
42 # Reference-style link
43 lines.append(f"See [Reference {i}][ref{i}]\n")
44 else:
45 # Plain text with potential regex traps
46 lines.append(f"This is paragraph {i} with some [text] that looks like links.\n")
47
48 # Add reference definitions at the end
49 for i in range(num_links):
50 if i % 3 == 1:
51 lines.append(f"[ref{i}]: https://example.com/ref/{i}\n")
52
53 return "".join(lines)
54
55if __name__ == "__main__":
56 profile_link_checker()執行方式：
1# Direct execution
2python profile_link_checker.py
3
4# Using cProfile from command line
5python -m cProfile -s cumulative markdown_link_checker.py --dir ./docs/步驟 2：使用 pstats 分析結果
pstats 模組提供更細緻的結果分析功能：
1import cProfile
2import pstats
3from pstats import SortKey
4
5def detailed_analysis():
6 """Perform detailed analysis with pstats"""
7
8 # Profile the code
9 profiler = cProfile.Profile()
10 profiler.enable()
11
12 # Run the target function
13 checker = MarkdownLinkChecker()
14 content = generate_test_content(500)
15 for _ in range(50):
16 checker.parse_markdown_links(content)
17
18 profiler.disable()
19
20 # Create Stats object
21 stats = pstats.Stats(profiler)
22
23 # Different sorting options
24 print("=" * 70)
25 print("Top 10 by CUMULATIVE time (including sub-calls)")
26 print("=" * 70)
27 stats.sort_stats(SortKey.CUMULATIVE).print_stats(10)
28
29 print("\n" + "=" * 70)
30 print("Top 10 by TOTAL time (excluding sub-calls)")
31 print("=" * 70)
32 stats.sort_stats(SortKey.TIME).print_stats(10)
33
34 print("\n" + "=" * 70)
35 print("Top 10 by CALL count")
36 print("=" * 70)
37 stats.sort_stats(SortKey.CALLS).print_stats(10)
38
39 # Filter by function name
40 print("\n" + "=" * 70)
41 print("Functions containing 'parse' or 'match'")
42 print("=" * 70)
43 stats.sort_stats(SortKey.TIME).print_stats('parse|match', 10)
44
45 # Show callers of a specific function
46 print("\n" + "=" * 70)
47 print("Who calls 'finditer'?")
48 print("=" * 70)
49 stats.print_callers('finditer')
50
51 # Save stats to file for later analysis
52 stats.dump_stats('link_checker.prof')
53
54 return stats
55
56def load_and_compare_profiles():
57 """Load and compare saved profile data"""
58 try:
59 # Load saved profile
60 stats = pstats.Stats('link_checker.prof')
61
62 # Add another profile for comparison
63 stats.add('link_checker_optimized.prof')
64
65 # Print combined stats
66 stats.sort_stats(SortKey.CUMULATIVE).print_stats(20)
67 except FileNotFoundError:
68 print("Profile files not found. Run detailed_analysis() first.")步驟 3：使用 line_profiler 進行行級分析
line_profiler 可以分析每一行程式碼的執行時間，適合精確定位瓶頸。
首先安裝 line_profiler：
1pip install line_profiler使用裝飾器標記要分析的函式：
1# profile_lines.py
2from line_profiler import profile
3
4@profile
5def parse_markdown_links_profiled(self, content: str):
6 """
7 Line-by-line profiled version of parse_markdown_links
8 """
9 links = []
10 lines = content.split('\n')
11
12 # Collect reference definitions
13 reference_defs = {}
14 for match in self.REFERENCE_DEF_PATTERN.finditer(content):
15 ref_name = match.group(1).lower()
16 ref_target = match.group(2).strip()
17 reference_defs[ref_name] = ref_target
18
19 in_code_block = False
20
21 for line_num, line in enumerate(lines, start=1):
22 if line.strip().startswith("```"):
23 in_code_block = not in_code_block
24 continue
25
26 if in_code_block:
27 continue
28
29 # These regex operations might be slow
30 for match in self.INLINE_LINK_PATTERN.finditer(line):
31 links.append({
32 "text": match.group(1),
33 "target": match.group(2),
34 "line": line_num
35 })
36
37 for match in self.REFERENCE_USE_PATTERN.finditer(line):
38 ref_name = match.group(2).lower()
39 if ref_name in reference_defs:
40 links.append({
41 "text": match.group(1),
42 "target": reference_defs[ref_name],
43 "line": line_num
44 })
45
46 return links
47
48# Alternative: Manual timing for specific sections
49import time
50
51def parse_with_timing(self, content: str):
52 """Manual timing for detailed analysis"""
53 timings = {}
54
55 # Time: split lines
56 start = time.perf_counter()
57 lines = content.split('\n')
58 timings['split_lines'] = time.perf_counter() - start
59
60 # Time: parse reference definitions
61 start = time.perf_counter()
62 reference_defs = {}
63 for match in self.REFERENCE_DEF_PATTERN.finditer(content):
64 ref_name = match.group(1).lower()
65 ref_target = match.group(2).strip()
66 reference_defs[ref_name] = ref_target
67 timings['parse_refs'] = time.perf_counter() - start
68
69 # Time: main parsing loop
70 start = time.perf_counter()
71 links = []
72 in_code_block = False
73
74 for line_num, line in enumerate(lines, start=1):
75 if line.strip().startswith("```"):
76 in_code_block = not in_code_block
77 continue
78
79 if in_code_block:
80 continue
81
82 for match in self.INLINE_LINK_PATTERN.finditer(line):
83 links.append({
84 "text": match.group(1),
85 "target": match.group(2),
86 "line": line_num
87 })
88
89 for match in self.REFERENCE_USE_PATTERN.finditer(line):
90 ref_name = match.group(2).lower()
91 if ref_name in reference_defs:
92 links.append({
93 "text": match.group(1),
94 "target": reference_defs[ref_name],
95 "line": line_num
96 })
97
98 timings['main_loop'] = time.perf_counter() - start
99
100 return links, timings執行 line_profiler：
1# Run with kernprof
2kernprof -l -v profile_lines.py
3
4# Or use the newer approach
5python -m line_profiler profile_lines.py步驟 4：分析正則表達式效能
正則表達式是常見的效能瓶頸，需要特別分析：
1import re
2import time
3from typing import Callable
4
5def benchmark_regex_patterns():
6 """Compare different regex pattern implementations"""
7
8 # Test content with various edge cases
9 test_lines = [
10 "Check out [Link](https://example.com)",
11 "See [Text with [brackets]](/python-advanced/04-cpython-internals/case-studies/profiling/url)",
12 "Multiple [link1](/python-advanced/04-cpython-internals/case-studies/profiling/url1) and [link2](/python-advanced/04-cpython-internals/case-studies/profiling/url2)",
13 "No links here, just plain text",
14 "Tricky [text](/python-advanced/04-cpython-internals/case-studies/profiling/url) with more [text](/python-advanced/04-cpython-internals/case-studies/profiling/url) links",
15 " should be ignored",
16 "[Link with spaces]( url with spaces )",
17 ] * 1000
18
19 test_content = "\n".join(test_lines)
20
21 # Pattern 1: Original pattern
22 pattern1 = re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)')
23
24 # Pattern 2: Possessive-like (using atomic group simulation)
25 # Note: Python re doesn't support possessive quantifiers directly
26 pattern2 = re.compile(r'(?<!!)\[([^\]]*)\]\(([^)]*)\)')
27
28 # Pattern 3: More specific pattern
29 pattern3 = re.compile(r'(?<!!)\[([^\[\]]+)\]\(([^()]+)\)')
30
31 # Pattern 4: Non-capturing groups where possible
32 pattern4 = re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)')
33
34 patterns = {
35 "original": pattern1,
36 "non_greedy": pattern2,
37 "more_specific": pattern3,
38 "optimized": pattern4,
39 }
40
41 results = {}
42
43 for name, pattern in patterns.items():
44 # Warmup
45 for _ in range(10):
46 list(pattern.finditer(test_content))
47
48 # Benchmark
49 start = time.perf_counter()
50 for _ in range(100):
51 matches = list(pattern.finditer(test_content))
52 elapsed = time.perf_counter() - start
53
54 results[name] = {
55 "time": elapsed,
56 "matches": len(matches),
57 }
58
59 # Print comparison
60 print("Regex Pattern Performance Comparison")
61 print("=" * 60)
62 print(f"{'Pattern':<20} {'Time (s)':<12} {'Matches':<10}")
63 print("-" * 60)
64
65 baseline = results["original"]["time"]
66 for name, data in results.items():
67 speedup = baseline / data["time"]
68 print(f"{name:<20} {data['time']:<12.4f} {data['matches']:<10} ({speedup:.2f}x)")
69
70 return results
71
72def analyze_regex_backtracking():
73 """Analyze potential regex backtracking issues"""
74 import re
75
76 # Patterns that might cause backtracking
77 problematic_patterns = [
78 (r'\[(.+)\]\((.+)\)', "Greedy .+ can backtrack"),
79 (r'\[([^\]]+)\]\(([^)]+)\)', "Negated class - better"),
80 (r'\[(.*?)\]\((.*?)\)', "Non-greedy - still may backtrack"),
81 ]
82
83 # Pathological input that triggers backtracking
84 pathological = "[" + "a" * 100 + "]" # No closing bracket pattern
85
86 print("Regex Backtracking Analysis")
87 print("=" * 60)
88
89 for pattern_str, description in problematic_patterns:
90 pattern = re.compile(pattern_str)
91
92 start = time.perf_counter()
93 try:
94 # Set a timeout using signal (Unix only)
95 result = pattern.search(pathological)
96 elapsed = time.perf_counter() - start
97 print(f"Pattern: {pattern_str}")
98 print(f" Description: {description}")
99 print(f" Time: {elapsed:.4f}s")
100 print(f" Match: {result}")
101 print()
102 except Exception as e:
103 print(f"Pattern: {pattern_str} - Error: {e}")
104
105def compare_compile_vs_inline():
106 """Compare precompiled vs inline regex performance"""
107
108 test_content = generate_test_content(500)
109 iterations = 100
110
111 # Test 1: Precompiled pattern (recommended)
112 compiled_pattern = re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)')
113
114 start = time.perf_counter()
115 for _ in range(iterations):
116 list(compiled_pattern.finditer(test_content))
117 compiled_time = time.perf_counter() - start
118
119 # Test 2: Inline pattern (compiled each time by re module cache)
120 start = time.perf_counter()
121 for _ in range(iterations):
122 list(re.finditer(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)', test_content))
123 inline_time = time.perf_counter() - start
124
125 # Test 3: Pattern compiled inside loop (worst case)
126 start = time.perf_counter()
127 for _ in range(iterations):
128 pattern = re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)')
129 list(pattern.finditer(test_content))
130 loop_compile_time = time.perf_counter() - start
131
132 print("Compile Strategy Comparison")
133 print("=" * 60)
134 print(f"{'Strategy':<25} {'Time (s)':<12} {'Relative':<10}")
135 print("-" * 60)
136 print(f"{'Precompiled (class)':<25} {compiled_time:<12.4f} {'1.00x':<10}")
137 print(f"{'Inline (re cache)':<25} {inline_time:<12.4f} {inline_time/compiled_time:.2f}x")
138 print(f"{'Compile in loop':<25} {loop_compile_time:<12.4f} {loop_compile_time/compiled_time:.2f}x")步驟 5：優化建議與驗證
根據分析結果，實作優化版本並驗證效果：
1import re
2from typing import List, Dict, Tuple
3from dataclasses import dataclass
4
5@dataclass(slots=True) # Python 3.10+ optimization
6class LinkInfo:
7 """Link information with memory-efficient storage"""
8 text: str
9 target: str
10 line: int
11
12class OptimizedLinkChecker:
13 """Optimized Markdown link checker based on profiling results"""
14
15 # Precompiled patterns at class level
16 INLINE_LINK_PATTERN = re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)')
17 REFERENCE_DEF_PATTERN = re.compile(r'^\s*\[([^\]]+)\]:\s*(.+)$', re.MULTILINE)
18 REFERENCE_USE_PATTERN = re.compile(r'\[([^\]]+)\]\[([^\]]+)\]')
19 CODE_BLOCK_PATTERN = re.compile(r'^```')
20
21 def parse_markdown_links_optimized(self, content: str) -> List[LinkInfo]:
22 """
23 Optimized link parsing with reduced memory allocation
24
25 Optimizations:
26 1. Use dataclass with slots for link storage
27 2. Single pass where possible
28 3. Avoid repeated string operations
29 4. Use local variables for frequently accessed attributes
30 """
31 links = []
32
33 # Cache pattern references for faster access
34 inline_pattern = self.INLINE_LINK_PATTERN
35 ref_use_pattern = self.REFERENCE_USE_PATTERN
36 ref_def_pattern = self.REFERENCE_DEF_PATTERN
37
38 # Collect reference definitions first (single pass over content)
39 reference_defs = {
40 match.group(1).lower(): match.group(2).strip()
41 for match in ref_def_pattern.finditer(content)
42 }
43
44 # Process line by line
45 in_code_block = False
46 line_start = 0
47
48 for line_num, line_end in enumerate(
49 self._find_line_positions(content), start=1
50 ):
51 line = content[line_start:line_end]
52
53 # Fast code block check
54 if line.lstrip().startswith("```"):
55 in_code_block = not in_code_block
56 line_start = line_end + 1
57 continue
58
59 if not in_code_block:
60 # Parse inline links
61 for match in inline_pattern.finditer(line):
62 links.append(LinkInfo(
63 text=match.group(1),
64 target=match.group(2),
65 line=line_num
66 ))
67
68 # Parse reference links
69 for match in ref_use_pattern.finditer(line):
70 ref_name = match.group(2).lower()
71 target = reference_defs.get(ref_name)
72 if target:
73 links.append(LinkInfo(
74 text=match.group(1),
75 target=target,
76 line=line_num
77 ))
78
79 line_start = line_end + 1
80
81 return links
82
83 def _find_line_positions(self, content: str):
84 """Generator that yields line end positions"""
85 pos = 0
86 while True:
87 newline = content.find('\n', pos)
88 if newline == -1:
89 yield len(content)
90 break
91 yield newline
92 pos = newline + 1
93
94def verify_optimization():
95 """Verify that optimizations maintain correctness and improve performance"""
96 import time
97
98 # Generate test content
99 test_content = generate_test_content(1000)
100
101 # Original implementation
102 original = MarkdownLinkChecker()
103
104 # Optimized implementation
105 optimized = OptimizedLinkChecker()
106
107 # Verify correctness
108 original_links = original.parse_markdown_links(test_content)
109 optimized_links = optimized.parse_markdown_links_optimized(test_content)
110
111 # Compare results
112 assert len(original_links) == len(optimized_links), \
113 f"Link count mismatch: {len(original_links)} vs {len(optimized_links)}"
114
115 for orig, opt in zip(original_links, optimized_links):
116 assert orig["text"] == opt.text, f"Text mismatch: {orig['text']} vs {opt.text}"
117 assert orig["target"] == opt.target, f"Target mismatch"
118 assert orig["line"] == opt.line, f"Line mismatch"
119
120 print("Correctness verified!")
121
122 # Benchmark
123 iterations = 100
124
125 # Original
126 start = time.perf_counter()
127 for _ in range(iterations):
128 original.parse_markdown_links(test_content)
129 original_time = time.perf_counter() - start
130
131 # Optimized
132 start = time.perf_counter()
133 for _ in range(iterations):
134 optimized.parse_markdown_links_optimized(test_content)
135 optimized_time = time.perf_counter() - start
136
137 print(f"\nPerformance Comparison ({iterations} iterations)")
138 print("=" * 50)
139 print(f"Original: {original_time:.4f}s")
140 print(f"Optimized: {optimized_time:.4f}s")
141 print(f"Speedup: {original_time / optimized_time:.2f}x")完整程式碼
以下是整合所有分析功能的完整腳本：
1#!/usr/bin/env python3
2"""
3Performance Profiling Script for Markdown Link Checker
4
5This script demonstrates various profiling techniques:
61. cProfile for function-level analysis
72. pstats for detailed statistics
83. line_profiler for line-by-line analysis
94. Regex pattern benchmarking
105. Optimization verification
11
12Usage:
13 python profiling_demo.py [--full|--quick|--regex|--verify]
14"""
15
16import argparse
17import cProfile
18import pstats
19import re
20import sys
21import time
22from dataclasses import dataclass
23from io import StringIO
24from pathlib import Path
25from pstats import SortKey
26from typing import Dict, List, Optional
27
28# ===== Original Implementation =====
29
30class MarkdownLinkChecker:
31 """Original Markdown link checker for comparison"""
32
33 INLINE_LINK_PATTERN = re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)')
34 REFERENCE_DEF_PATTERN = re.compile(r'^\s*\[([^\]]+)\]:\s*(.+)$', re.MULTILINE)
35 REFERENCE_USE_PATTERN = re.compile(r'\[([^\]]+)\]\[([^\]]+)\]')
36
37 def parse_markdown_links(self, content: str) -> List[Dict]:
38 links = []
39 lines = content.split('\n')
40
41 reference_defs = {}
42 for match in self.REFERENCE_DEF_PATTERN.finditer(content):
43 ref_name = match.group(1).lower()
44 ref_target = match.group(2).strip()
45 reference_defs[ref_name] = ref_target
46
47 in_code_block = False
48
49 for line_num, line in enumerate(lines, start=1):
50 if line.strip().startswith("```"):
51 in_code_block = not in_code_block
52 continue
53
54 if in_code_block:
55 continue
56
57 for match in self.INLINE_LINK_PATTERN.finditer(line):
58 links.append({
59 "text": match.group(1),
60 "target": match.group(2),
61 "line": line_num
62 })
63
64 for match in self.REFERENCE_USE_PATTERN.finditer(line):
65 ref_name = match.group(2).lower()
66 if ref_name in reference_defs:
67 links.append({
68 "text": match.group(1),
69 "target": reference_defs[ref_name],
70 "line": line_num
71 })
72
73 return links
74
75# ===== Test Data Generation =====
76
77def generate_test_content(num_links: int) -> str:
78 """Generate test Markdown content"""
79 lines = ["# Test Document\n\n"]
80
81 for i in range(num_links):
82 if i % 3 == 0:
83 lines.append(f"Check out [Link {i}](https://example.com/{i})\n")
84 elif i % 3 == 1:
85 lines.append(f"See [Reference {i}][ref{i}]\n")
86 else:
87 lines.append(f"Paragraph {i} with some text.\n")
88
89 lines.append("\n")
90 for i in range(num_links):
91 if i % 3 == 1:
92 lines.append(f"[ref{i}]: https://example.com/ref/{i}\n")
93
94 return "".join(lines)
95
96# ===== Profiling Functions =====
97
98def run_cprofile_analysis(iterations: int = 50, num_links: int = 500):
99 """Run cProfile analysis"""
100 print("Running cProfile Analysis")
101 print("=" * 70)
102
103 content = generate_test_content(num_links)
104 checker = MarkdownLinkChecker()
105
106 profiler = cProfile.Profile()
107 profiler.enable()
108
109 for _ in range(iterations):
110 checker.parse_markdown_links(content)
111
112 profiler.disable()
113
114 stats = pstats.Stats(profiler)
115 stats.sort_stats(SortKey.CUMULATIVE)
116
117 print(f"\nTop 15 functions by cumulative time:")
118 print("-" * 70)
119 stats.print_stats(15)
120
121 return stats
122
123def run_regex_benchmark():
124 """Benchmark different regex patterns"""
125 print("\nRegex Pattern Benchmark")
126 print("=" * 70)
127
128 content = generate_test_content(500)
129
130 patterns = {
131 "Original": re.compile(r'(?<!!)\[([^\]]+)\]\(([^)]+)\)'),
132 "Non-greedy inner": re.compile(r'(?<!!)\[([^\]]*?)\]\(([^)]*?)\)'),
133 "Anchored": re.compile(r'(?<!!)\[([^\[\]]+)\]\(([^()]+)\)'),
134 }
135
136 print(f"{'Pattern':<25} {'Time (ms)':<12} {'Matches':<10}")
137 print("-" * 50)
138
139 for name, pattern in patterns.items():
140 # Warmup
141 list(pattern.finditer(content))
142
143 # Benchmark
144 start = time.perf_counter()
145 for _ in range(100):
146 matches = list(pattern.finditer(content))
147 elapsed = (time.perf_counter() - start) * 1000
148
149 print(f"{name:<25} {elapsed:<12.2f} {len(matches):<10}")
150
151def run_compile_comparison():
152 """Compare precompiled vs inline regex"""
153 print("\nCompile Strategy Comparison")
154 print("=" * 70)
155
156 content = generate_test_content(500)
157 iterations = 100
158 pattern_str = r'(?<!!)\[([^\]]+)\]\(([^)]+)\)'
159
160 # Precompiled
161 compiled = re.compile(pattern_str)
162 start = time.perf_counter()
163 for _ in range(iterations):
164 list(compiled.finditer(content))
165 compiled_time = time.perf_counter() - start
166
167 # Inline (uses re module cache)
168 start = time.perf_counter()
169 for _ in range(iterations):
170 list(re.finditer(pattern_str, content))
171 inline_time = time.perf_counter() - start
172
173 print(f"{'Strategy':<25} {'Time (ms)':<12} {'Relative':<10}")
174 print("-" * 50)
175 print(f"{'Precompiled':<25} {compiled_time*1000:<12.2f} {'1.00x':<10}")
176 print(f"{'Inline (cached)':<25} {inline_time*1000:<12.2f} {inline_time/compiled_time:.2f}x")
177
178def main():
179 parser = argparse.ArgumentParser(description="Profiling Demo")
180 parser.add_argument('--full', action='store_true', help='Run full analysis')
181 parser.add_argument('--quick', action='store_true', help='Run quick analysis')
182 parser.add_argument('--regex', action='store_true', help='Run regex benchmark')
183 args = parser.parse_args()
184
185 if args.regex:
186 run_regex_benchmark()
187 run_compile_comparison()
188 elif args.quick:
189 run_cprofile_analysis(iterations=10, num_links=100)
190 else:
191 run_cprofile_analysis()
192 run_regex_benchmark()
193 run_compile_comparison()
194
195if __name__ == "__main__":
196 main()分析結果範例
執行 cProfile 分析後的典型輸出：
1Running cProfile Analysis
2======================================================================
3
4Top 15 functions by cumulative time:
5----------------------------------------------------------------------
6 125003 function calls in 0.892 seconds
7
8 Ordered by: cumulative time
9
10 ncalls tottime percall cumtime percall filename:lineno(function)
11 50 0.012 0.000 0.892 0.018 checker.py:45(parse_markdown_links)
12 25000 0.234 0.000 0.567 0.000 {method 'finditer' of 're.Pattern'}
13 50 0.089 0.002 0.089 0.002 {method 'split' of 'str' objects}
14 50000 0.156 0.000 0.156 0.000 {method 'append' of 'list' objects}
15 25000 0.098 0.000 0.098 0.000 {method 'group' of 're.Match'}
16 12500 0.045 0.000 0.045 0.000 {method 'lower' of 'str' objects}
17 12500 0.034 0.000 0.034 0.000 {method 'strip' of 'str' objects}
18 25000 0.067 0.000 0.067 0.000 {method 'startswith' of 'str' objects}
19 50 0.023 0.000 0.023 0.000 {built-in method builtins.enumerate}從上述結果可以觀察到：
finditer佔用最多時間：正則表達式匹配是主要瓶頸split操作耗時明顯：每次解析都建立新的行列表append呼叫頻繁：大量的字典建立和列表操作
line_profiler 的典型輸出：
1Timer unit: 1e-06 s
2
3Total time: 0.456 s
4File: checker.py
5Function: parse_markdown_links at line 45
6
7Line # Hits Time Per Hit % Time Line Contents
8==============================================================
9 45 def parse_markdown_links(self, content):
10 46 50 1234.0 24.7 0.3 links = []
11 47 50 89012.0 1780.2 19.5 lines = content.split('\n')
12 48
13 49 50 56.0 1.1 0.0 reference_defs = {}
14 50 2500 45678.0 18.3 10.0 for match in self.REFERENCE_DEF_PATTERN.finditer(content):
15 51 2450 3456.0 1.4 0.8 ref_name = match.group(1).lower()
16 52 2450 2345.0 1.0 0.5 ref_target = match.group(2).strip()
17 53 2450 1234.0 0.5 0.3 reference_defs[ref_name] = ref_target
18 54
19 55 50 34.0 0.7 0.0 in_code_block = False
20 56
21 57 25050 12345.0 0.5 2.7 for line_num, line in enumerate(lines, start=1):
22 58 25000 34567.0 1.4 7.6 if line.strip().startswith("```"):
23 59 in_code_block = not in_code_block
24 60 continue
25 61
26 62 25000 5678.0 0.2 1.2 if in_code_block:
27 63 continue
28 64
29 65 25000 156789.0 6.3 34.4 for match in self.INLINE_LINK_PATTERN.finditer(line):
30 66 8350 23456.0 2.8 5.1 links.append({...})
31 67
32 68 25000 78901.0 3.2 17.3 for match in self.REFERENCE_USE_PATTERN.finditer(line):
33 69 2450 1234.0 0.5 0.3 ref_name = match.group(2).lower()
34 70 2450 567.0 0.2 0.1 if ref_name in reference_defs:
35 71 2450 1234.0 0.5 0.3 links.append({...})
36 72
37 73 50 23.0 0.5 0.0 return links關鍵發現：
- 第 65 行（行內連結匹配）佔 34.4%：這是最大的瓶頸
- 第 47 行（split）佔 19.5%：字串分割是第二大消耗
- 第 68 行（引用連結匹配）佔 17.3%：也是重要的優化目標
設計權衡
| 面向 | 不分析 | 使用 cProfile | 使用 line_profiler |
|---|---|---|---|
| 開發時間 | 少 | 中等 | 較多 |
| 分析粒度 | 無 | 函式級 | 行級 |
| 效能開銷 | 無 | 低 | 較高 |
| 適用場景 | 簡單程式 | 一般優化 | 精確定位 |
| 學習成本 | 無 | 低 | 中等 |
| 結果準確度 | - | 高 | 非常高 |
什麼時候該做效能分析？
適合分析：
- 程式明顯變慢
- 處理大量資料
- 發布前的效能驗證
- 正則表達式複雜時
- 有迴圈處理大量項目
不建議過早優化：
- 功能還在開發中
- 使用頻率很低
- 效能已經足夠
- 可讀性更重要時
練習
基礎練習：用 cProfile 分析排序函式
1"""
2Exercise 1: Profile different sorting approaches
3"""
4import cProfile
5import pstats
6import random
7from pstats import SortKey
8
9def bubble_sort(arr):
10 """Bubble sort implementation"""
11 n = len(arr)
12 arr = arr.copy()
13 for i in range(n):
14 for j in range(0, n - i - 1):
15 if arr[j] > arr[j + 1]:
16 arr[j], arr[j + 1] = arr[j + 1], arr[j]
17 return arr
18
19def quick_sort(arr):
20 """Quick sort implementation"""
21 if len(arr) <= 1:
22 return arr
23 pivot = arr[len(arr) // 2]
24 left = [x for x in arr if x < pivot]
25 middle = [x for x in arr if x == pivot]
26 right = [x for x in arr if x > pivot]
27 return quick_sort(left) + middle + quick_sort(right)
28
29def profile_sorting():
30 """Profile and compare sorting algorithms"""
31 data = [random.randint(0, 10000) for _ in range(1000)]
32
33 # Profile bubble sort
34 profiler = cProfile.Profile()
35 profiler.enable()
36 bubble_sort(data)
37 profiler.disable()
38
39 print("Bubble Sort Profile:")
40 stats = pstats.Stats(profiler)
41 stats.sort_stats(SortKey.TIME).print_stats(5)
42
43 # Profile quick sort
44 profiler = cProfile.Profile()
45 profiler.enable()
46 quick_sort(data)
47 profiler.disable()
48
49 print("\nQuick Sort Profile:")
50 stats = pstats.Stats(profiler)
51 stats.sort_stats(SortKey.TIME).print_stats(5)
52
53if __name__ == "__main__":
54 profile_sorting()進階練習：用 line_profiler 找出熱點程式碼
1"""
2Exercise 2: Use line_profiler to find hotspots
3
4Instructions:
51. Install line_profiler: pip install line_profiler
62. Add @profile decorator to functions you want to analyze
73. Run: kernprof -l -v exercise2.py
8"""
9from line_profiler import profile
10
11@profile
12def find_primes(n):
13 """Find all prime numbers up to n"""
14 primes = []
15 for num in range(2, n + 1):
16 is_prime = True
17 for i in range(2, int(num ** 0.5) + 1):
18 if num % i == 0:
19 is_prime = False
20 break
21 if is_prime:
22 primes.append(num)
23 return primes
24
25@profile
26def find_primes_sieve(n):
27 """Find primes using Sieve of Eratosthenes"""
28 sieve = [True] * (n + 1)
29 sieve[0] = sieve[1] = False
30
31 for i in range(2, int(n ** 0.5) + 1):
32 if sieve[i]:
33 for j in range(i * i, n + 1, i):
34 sieve[j] = False
35
36 return [i for i, is_prime in enumerate(sieve) if is_prime]
37
38if __name__ == "__main__":
39 print("Finding primes up to 10000...")
40 primes1 = find_primes(10000)
41 primes2 = find_primes_sieve(10000)
42 print(f"Found {len(primes1)} primes (basic)")
43 print(f"Found {len(primes2)} primes (sieve)")挑戰題：比較不同正則表達式寫法的效能差異
1"""
2Exercise 3: Compare regex pattern performance
3
4Task: Parse email addresses from text using different patterns
5and measure their performance.
6"""
7import re
8import time
9
10def benchmark_email_patterns():
11 """Compare different email regex patterns"""
12
13 # Test content with mixed valid and invalid emails
14 test_text = """
15 Contact us at support@example.com or sales@company.org
16 Invalid: not.an.email, @missing.com, missing@
17 Valid: user.name+tag@domain.co.uk, test123@sub.domain.com
18 Edge cases: "quoted"@domain.com, user@[192.168.1.1]
19 """ * 1000
20
21 patterns = {
22 # Simple pattern (may miss some valid emails)
23 "simple": re.compile(r'\b[\w.-]+@[\w.-]+\.\w+\b'),
24
25 # More comprehensive pattern
26 "comprehensive": re.compile(
27 r'\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b'
28 ),
29
30 # RFC 5322 inspired (complex but thorough)
31 "rfc_inspired": re.compile(
32 r'(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*'
33 r'|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]'
34 r'|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")'
35 r'@(?:(?:[a-z0-9](/python-advanced/04-cpython-internals/case-studies/profiling/?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](/python-advanced/04-cpython-internals/case-studies/profiling/?:[a-z0-9-]*[a-z0-9])?'
36 r'|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}'
37 r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:'
38 r'(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]'
39 r'|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])',
40 re.IGNORECASE
41 ),
42 }
43
44 print("Email Pattern Performance Comparison")
45 print("=" * 60)
46 print(f"{'Pattern':<20} {'Time (ms)':<12} {'Matches':<10}")
47 print("-" * 60)
48
49 for name, pattern in patterns.items():
50 # Warmup
51 list(pattern.findall(test_text))
52
53 # Benchmark
54 start = time.perf_counter()
55 for _ in range(100):
56 matches = pattern.findall(test_text)
57 elapsed = (time.perf_counter() - start) * 1000
58
59 print(f"{name:<20} {elapsed:<12.2f} {len(matches):<10}")
60
61 # Your task: Add more patterns and analyze the results
62 # Consider: What trade-offs exist between accuracy and speed?
63
64if __name__ == "__main__":
65 benchmark_email_patterns()延伸閱讀
下一章：記憶體優化