projects/linux·blob 0d55ec6f · refs

451 lines · 13.1 KiB

1# export-to-sqlite.py: export perf data to a sqlite3 database
2# Copyright (c) 2017, Intel Corporation.
3#
4# This program is free software; you can redistribute it and/or modify it
5# under the terms and conditions of the GNU General Public License,
6# version 2, as published by the Free Software Foundation.
7#
8# This program is distributed in the hope it will be useful, but WITHOUT
9# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11# more details.
12
13import os
14import sys
15import struct
16import datetime
17
18# To use this script you will need to have installed package python-pyside which
19# provides LGPL-licensed Python bindings for Qt. You will also need the package
20# libqt4-sql-sqlite for Qt sqlite3 support.
21#
22# An example of using this script with Intel PT:
23#
24# $ perf record -e intel_pt//u ls
25# $ perf script -s ~/libexec/perf-core/scripts/python/export-to-sqlite.py pt_example branches calls
26# 2017-07-31 14:26:07.326913 Creating database...
27# 2017-07-31 14:26:07.538097 Writing records...
28# 2017-07-31 14:26:09.889292 Adding indexes
29# 2017-07-31 14:26:09.958746 Done
30#
31# To browse the database, sqlite3 can be used e.g.
32#
33# $ sqlite3 pt_example
34# sqlite> .header on
35# sqlite> select * from samples_view where id < 10;
36# sqlite> .mode column
37# sqlite> select * from samples_view where id < 10;
38# sqlite> .tables
39# sqlite> .schema samples_view
40# sqlite> .quit
41#
42# An example of using the database is provided by the script
43# call-graph-from-sql.py. Refer to that script for details.
44#
45# The database structure is practically the same as created by the script
46# export-to-postgresql.py. Refer to that script for details. A notable
47# difference is the 'transaction' column of the 'samples' table which is
48# renamed 'transaction_' in sqlite because 'transaction' is a reserved word.
49
50from PySide.QtSql import *
51
52sys.path.append(os.environ['PERF_EXEC_PATH'] + \
53 '/scripts/python/Perf-Trace-Util/lib/Perf/Trace')
54
55# These perf imports are not used at present
56#from perf_trace_context import *
57#from Core import *
58
59perf_db_export_mode = True
60perf_db_export_calls = False
61perf_db_export_callchains = False
62
63def usage():
64 print >> sys.stderr, "Usage is: export-to-sqlite.py <database name> [<columns>] [<calls>] [<callchains>]"
65 print >> sys.stderr, "where: columns 'all' or 'branches'"
66 print >> sys.stderr, " calls 'calls' => create calls and call_paths table"
67 print >> sys.stderr, " callchains 'callchains' => create call_paths table"
68 raise Exception("Too few arguments")
69
70if (len(sys.argv) < 2):
71 usage()
72
73dbname = sys.argv[1]
74
75if (len(sys.argv) >= 3):
76 columns = sys.argv[2]
77else:
78 columns = "all"
79
80if columns not in ("all", "branches"):
81 usage()
82
83branches = (columns == "branches")
84
85for i in range(3,len(sys.argv)):
86 if (sys.argv[i] == "calls"):
87 perf_db_export_calls = True
88 elif (sys.argv[i] == "callchains"):
89 perf_db_export_callchains = True
90 else:
91 usage()
92
93def do_query(q, s):
94 if (q.exec_(s)):
95 return
96 raise Exception("Query failed: " + q.lastError().text())
97
98def do_query_(q):
99 if (q.exec_()):
100 return
101 raise Exception("Query failed: " + q.lastError().text())
102
103print datetime.datetime.today(), "Creating database..."
104
105db_exists = False
106try:
107 f = open(dbname)
108 f.close()
109 db_exists = True
110except:
111 pass
112
113if db_exists:
114 raise Exception(dbname + " already exists")
115
116db = QSqlDatabase.addDatabase('QSQLITE')
117db.setDatabaseName(dbname)
118db.open()
119
120query = QSqlQuery(db)
121
122do_query(query, 'PRAGMA journal_mode = OFF')
123do_query(query, 'BEGIN TRANSACTION')
124
125do_query(query, 'CREATE TABLE selected_events ('
126 'id integer NOT NULL PRIMARY KEY,'
127 'name varchar(80))')
128do_query(query, 'CREATE TABLE machines ('
129 'id integer NOT NULL PRIMARY KEY,'
130 'pid integer,'
131 'root_dir varchar(4096))')
132do_query(query, 'CREATE TABLE threads ('
133 'id integer NOT NULL PRIMARY KEY,'
134 'machine_id bigint,'
135 'process_id bigint,'
136 'pid integer,'
137 'tid integer)')
138do_query(query, 'CREATE TABLE comms ('
139 'id integer NOT NULL PRIMARY KEY,'
140 'comm varchar(16))')
141do_query(query, 'CREATE TABLE comm_threads ('
142 'id integer NOT NULL PRIMARY KEY,'
143 'comm_id bigint,'
144 'thread_id bigint)')
145do_query(query, 'CREATE TABLE dsos ('
146 'id integer NOT NULL PRIMARY KEY,'
147 'machine_id bigint,'
148 'short_name varchar(256),'
149 'long_name varchar(4096),'
150 'build_id varchar(64))')
151do_query(query, 'CREATE TABLE symbols ('
152 'id integer NOT NULL PRIMARY KEY,'
153 'dso_id bigint,'
154 'sym_start bigint,'
155 'sym_end bigint,'
156 'binding integer,'
157 'name varchar(2048))')
158do_query(query, 'CREATE TABLE branch_types ('
159 'id integer NOT NULL PRIMARY KEY,'
160 'name varchar(80))')
161
162if branches:
163 do_query(query, 'CREATE TABLE samples ('
164 'id integer NOT NULL PRIMARY KEY,'
165 'evsel_id bigint,'
166 'machine_id bigint,'
167 'thread_id bigint,'
168 'comm_id bigint,'
169 'dso_id bigint,'
170 'symbol_id bigint,'
171 'sym_offset bigint,'
172 'ip bigint,'
173 'time bigint,'
174 'cpu integer,'
175 'to_dso_id bigint,'
176 'to_symbol_id bigint,'
177 'to_sym_offset bigint,'
178 'to_ip bigint,'
179 'branch_type integer,'
180 'in_tx boolean,'
181 'call_path_id bigint)')
182else:
183 do_query(query, 'CREATE TABLE samples ('
184 'id integer NOT NULL PRIMARY KEY,'
185 'evsel_id bigint,'
186 'machine_id bigint,'
187 'thread_id bigint,'
188 'comm_id bigint,'
189 'dso_id bigint,'
190 'symbol_id bigint,'
191 'sym_offset bigint,'
192 'ip bigint,'
193 'time bigint,'
194 'cpu integer,'
195 'to_dso_id bigint,'
196 'to_symbol_id bigint,'
197 'to_sym_offset bigint,'
198 'to_ip bigint,'
199 'period bigint,'
200 'weight bigint,'
201 'transaction_ bigint,'
202 'data_src bigint,'
203 'branch_type integer,'
204 'in_tx boolean,'
205 'call_path_id bigint)')
206
207if perf_db_export_calls or perf_db_export_callchains:
208 do_query(query, 'CREATE TABLE call_paths ('
209 'id integer NOT NULL PRIMARY KEY,'
210 'parent_id bigint,'
211 'symbol_id bigint,'
212 'ip bigint)')
213if perf_db_export_calls:
214 do_query(query, 'CREATE TABLE calls ('
215 'id integer NOT NULL PRIMARY KEY,'
216 'thread_id bigint,'
217 'comm_id bigint,'
218 'call_path_id bigint,'
219 'call_time bigint,'
220 'return_time bigint,'
221 'branch_count bigint,'
222 'call_id bigint,'
223 'return_id bigint,'
224 'parent_call_path_id bigint,'
225 'flags integer)')
226
227# printf was added to sqlite in version 3.8.3
228sqlite_has_printf = False
229try:
230 do_query(query, 'SELECT printf("") FROM machines')
231 sqlite_has_printf = True
232except:
233 pass
234
235def emit_to_hex(x):
236 if sqlite_has_printf:
237 return 'printf("%x", ' + x + ')'
238 else:
239 return x
240
241do_query(query, 'CREATE VIEW machines_view AS '
242 'SELECT '
243 'id,'
244 'pid,'
245 'root_dir,'
246 'CASE WHEN id=0 THEN \'unknown\' WHEN pid=-1 THEN \'host\' ELSE \'guest\' END AS host_or_guest'
247 ' FROM machines')
248
249do_query(query, 'CREATE VIEW dsos_view AS '
250 'SELECT '
251 'id,'
252 'machine_id,'
253 '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
254 'short_name,'
255 'long_name,'
256 'build_id'
257 ' FROM dsos')
258
259do_query(query, 'CREATE VIEW symbols_view AS '
260 'SELECT '
261 'id,'
262 'name,'
263 '(SELECT short_name FROM dsos WHERE id=dso_id) AS dso,'
264 'dso_id,'
265 'sym_start,'
266 'sym_end,'
267 'CASE WHEN binding=0 THEN \'local\' WHEN binding=1 THEN \'global\' ELSE \'weak\' END AS binding'
268 ' FROM symbols')
269
270do_query(query, 'CREATE VIEW threads_view AS '
271 'SELECT '
272 'id,'
273 'machine_id,'
274 '(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest,'
275 'process_id,'
276 'pid,'
277 'tid'
278 ' FROM threads')
279
280do_query(query, 'CREATE VIEW comm_threads_view AS '
281 'SELECT '
282 'comm_id,'
283 '(SELECT comm FROM comms WHERE id = comm_id) AS command,'
284 'thread_id,'
285 '(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
286 '(SELECT tid FROM threads WHERE id = thread_id) AS tid'
287 ' FROM comm_threads')
288
289if perf_db_export_calls or perf_db_export_callchains:
290 do_query(query, 'CREATE VIEW call_paths_view AS '
291 'SELECT '
292 'c.id,'
293 + emit_to_hex('c.ip') + ' AS ip,'
294 'c.symbol_id,'
295 '(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol,'
296 '(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id,'
297 '(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name,'
298 'c.parent_id,'
299 + emit_to_hex('p.ip') + ' AS parent_ip,'
300 'p.symbol_id AS parent_symbol_id,'
301 '(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol,'
302 '(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id,'
303 '(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name'
304 ' FROM call_paths c INNER JOIN call_paths p ON p.id = c.parent_id')
305if perf_db_export_calls:
306 do_query(query, 'CREATE VIEW calls_view AS '
307 'SELECT '
308 'calls.id,'
309 'thread_id,'
310 '(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
311 '(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
312 '(SELECT comm FROM comms WHERE id = comm_id) AS command,'
313 'call_path_id,'
314 + emit_to_hex('ip') + ' AS ip,'
315 'symbol_id,'
316 '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
317 'call_time,'
318 'return_time,'
319 'return_time - call_time AS elapsed_time,'
320 'branch_count,'
321 'call_id,'
322 'return_id,'
323 'CASE WHEN flags=1 THEN \'no call\' WHEN flags=2 THEN \'no return\' WHEN flags=3 THEN \'no call/return\' ELSE \'\' END AS flags,'
324 'parent_call_path_id'
325 ' FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id')
326
327do_query(query, 'CREATE VIEW samples_view AS '
328 'SELECT '
329 'id,'
330 'time,'
331 'cpu,'
332 '(SELECT pid FROM threads WHERE id = thread_id) AS pid,'
333 '(SELECT tid FROM threads WHERE id = thread_id) AS tid,'
334 '(SELECT comm FROM comms WHERE id = comm_id) AS command,'
335 '(SELECT name FROM selected_events WHERE id = evsel_id) AS event,'
336 + emit_to_hex('ip') + ' AS ip_hex,'
337 '(SELECT name FROM symbols WHERE id = symbol_id) AS symbol,'
338 'sym_offset,'
339 '(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name,'
340 + emit_to_hex('to_ip') + ' AS to_ip_hex,'
341 '(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol,'
342 'to_sym_offset,'
343 '(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name,'
344 '(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name,'
345 'in_tx'
346 ' FROM samples')
347
348do_query(query, 'END TRANSACTION')
349
350evsel_query = QSqlQuery(db)
351evsel_query.prepare("INSERT INTO selected_events VALUES (?, ?)")
352machine_query = QSqlQuery(db)
353machine_query.prepare("INSERT INTO machines VALUES (?, ?, ?)")
354thread_query = QSqlQuery(db)
355thread_query.prepare("INSERT INTO threads VALUES (?, ?, ?, ?, ?)")
356comm_query = QSqlQuery(db)
357comm_query.prepare("INSERT INTO comms VALUES (?, ?)")
358comm_thread_query = QSqlQuery(db)
359comm_thread_query.prepare("INSERT INTO comm_threads VALUES (?, ?, ?)")
360dso_query = QSqlQuery(db)
361dso_query.prepare("INSERT INTO dsos VALUES (?, ?, ?, ?, ?)")
362symbol_query = QSqlQuery(db)
363symbol_query.prepare("INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?)")
364branch_type_query = QSqlQuery(db)
365branch_type_query.prepare("INSERT INTO branch_types VALUES (?, ?)")
366sample_query = QSqlQuery(db)
367if branches:
368 sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
369else:
370 sample_query.prepare("INSERT INTO samples VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
371if perf_db_export_calls or perf_db_export_callchains:
372 call_path_query = QSqlQuery(db)
373 call_path_query.prepare("INSERT INTO call_paths VALUES (?, ?, ?, ?)")
374if perf_db_export_calls:
375 call_query = QSqlQuery(db)
376 call_query.prepare("INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
377
378def trace_begin():
379 print datetime.datetime.today(), "Writing records..."
380 do_query(query, 'BEGIN TRANSACTION')
381 # id == 0 means unknown. It is easier to create records for them than replace the zeroes with NULLs
382 evsel_table(0, "unknown")
383 machine_table(0, 0, "unknown")
384 thread_table(0, 0, 0, -1, -1)
385 comm_table(0, "unknown")
386 dso_table(0, 0, "unknown", "unknown", "")
387 symbol_table(0, 0, 0, 0, 0, "unknown")
388 sample_table(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
389 if perf_db_export_calls or perf_db_export_callchains:
390 call_path_table(0, 0, 0, 0)
391
392unhandled_count = 0
393
394def trace_end():
395 do_query(query, 'END TRANSACTION')
396
397 print datetime.datetime.today(), "Adding indexes"
398 if perf_db_export_calls:
399 do_query(query, 'CREATE INDEX pcpid_idx ON calls (parent_call_path_id)')
400
401 if (unhandled_count):
402 print datetime.datetime.today(), "Warning: ", unhandled_count, " unhandled events"
403 print datetime.datetime.today(), "Done"
404
405def trace_unhandled(event_name, context, event_fields_dict):
406 global unhandled_count
407 unhandled_count += 1
408
409def sched__sched_switch(*x):
410 pass
411
412def bind_exec(q, n, x):
413 for xx in x[0:n]:
414 q.addBindValue(str(xx))
415 do_query_(q)
416
417def evsel_table(*x):
418 bind_exec(evsel_query, 2, x)
419
420def machine_table(*x):
421 bind_exec(machine_query, 3, x)
422
423def thread_table(*x):
424 bind_exec(thread_query, 5, x)
425
426def comm_table(*x):
427 bind_exec(comm_query, 2, x)
428
429def comm_thread_table(*x):
430 bind_exec(comm_thread_query, 3, x)
431
432def dso_table(*x):
433 bind_exec(dso_query, 5, x)
434
435def symbol_table(*x):
436 bind_exec(symbol_query, 6, x)
437
438def branch_type_table(*x):
439 bind_exec(branch_type_query, 2, x)
440
441def sample_table(*x):
442 if branches:
443 bind_exec(sample_query, 18, x)
444 else:
445 bind_exec(sample_query, 22, x)
446
447def call_path_table(*x):
448 bind_exec(call_path_query, 4, x)
449
450def call_return_table(*x):
451 bind_exec(call_query, 11, x)