.fit file data munging
Posted on Mon 19 February 2018 in Projects
New coolest unit: semicircles
- Conversion: semicircles = degrees / ( 2^31 / 180 )
- 32-bit unsigned integer represents full 360 deg of longitude
- maximum precision from 32 bits (~double that of floating point), and integer arithmetic
details: https://msdn.microsoft.com/en-us/library/cc510650.aspx
.fit file parsing, data munging¶
In [1]:
from fitparse import FitFile, FitParseError
import pandas as pd
import gmaps
import os
import matplotlib.pyplot as plt
from matplotlib import cm
import sys
In [2]:
try:
fitfile = FitFile('2363427903.fit')
fitfile.parse()
except FitParseError, e:
print "Error while parsing .FIT file: %s" % e
sys.exit(1)
In [ ]:
fitfile.messages
In [3]:
fitfile.profile_version
Out[3]:
In [4]:
fitfile.protocol_version
Out[4]:
In [5]:
fitfile.messages[0].name
Out[5]:
In [6]:
fitfile.messages[0].get_values()
Out[6]:
In [7]:
fitfile.messages[0].get_value('garmin_product')
Out[7]:
In [8]:
fitfile.messages[9].get('altitude').value
Out[8]:
In [9]:
fitfile.messages[9].get('altitude').units
Out[9]:
In [10]:
# enumerate samples of all recorded data
gathered_names = []
d = {}
frames = []
for i in xrange(len(fitfile.messages)):
if fitfile.messages[i].name not in gathered_names:
gathered_names.append(fitfile.messages[i].name)
d = fitfile.messages[i].get_values()
frames.append(pd.DataFrame.from_dict(d, orient = 'index'))
df = pd.concat(frames, keys=gathered_names)
df.columns= ["Data Available (if multiple entries, only 1st is shown)"]
from IPython.display import display
with pd.option_context('display.max_rows', None, 'display.max_columns', None):
display(df)
In [11]:
# for plotting, clean up the data from 'record' (ie. lat, lon, altitude, distance, cadence, speed)
d = [] # initialize dict
r = 0 # initialize record counter
# degrees = semicircles * ( 180 / 2^31 )
# semicircles = degrees * ( 2^31 / 180 )
def deg(s):
return s*(180./(2**31))
# Get all data messages that are of type record
for record in fitfile.get_messages('record'):
r += 1
# Go through all the data entries in this record
for record_data in record:
if record_data.name in ['position_lat','position_long']: # if data is a lat or lon..
d.append((r, record_data.name, deg(record_data.value), 'deg')) # convert semicircles to degrees
else:
d.append((r, record_data.name, record_data.value, record_data.units))
df = pd.DataFrame(d, columns=('record','name', 'value','units'))
dfmini = df[df['record'] == 1]
df.head(12)
Out[11]:
In [12]:
df['value'][6] # full GPS precision is maintained on converted lats, lons
Out[12]:
In [13]:
cadence = df[df['name'].str.match('cadence')].reset_index().filter(regex='value').rename(columns={'value':'cadence'})
altitude = df[df['name'].str.match('altitude')].reset_index().filter(regex='value').rename(columns={'value':'altitude'})
distance = df[df['name'].str.match('distance')].reset_index().filter(regex='value').rename(columns={'value':'distance'})
speed = df[df['name'].str.match('speed')].reset_index().filter(regex='value').rename(columns={'value':'speed'})
lats = df[df['name'].str.match('position_lat')].reset_index().filter(regex='value').rename(columns={'value':'lat'})
lons = df[df['name'].str.match('position_long')].reset_index().filter(regex='value').rename(columns={'value':'lon'})
locs = pd.concat([lats, lons], axis=1)
locs.head()
Out[13]:
Plot stuff¶
In [14]:
filename = '2363427903.fit'
filename.rsplit('.', 1)[0].lower()
Out[14]:
In [15]:
plt.figure(figsize=(9,6))
plt.subplot(3, 1, 1)
plt.plot(distance/1600, cadence)
plt.axis([0, distance.max()[0]/1600, cadence.mean()[0]*0.95, cadence.mean()[0]*1.05])
plt.axhline(y=cadence.mean()[0], xmin=0, xmax=1, c='r', label = 'mean = ' + str(cadence.mean()[0].round(2)), ls='-', lw=2)
plt.xlabel('distance (mi)')
plt.ylabel('cadence')
# plt.title('cadence')
plt.legend(loc='best')
plt.subplot(3, 1, 2)
plt.plot(distance/1600, altitude)
plt.axis([0, distance.max()[0]/1600, altitude.min()[0], altitude.max()[0]])
plt.axhline(y=altitude.mean()[0], xmin=0, xmax=1, c='r', label = 'mean = ' + str(altitude.mean()[0].round(2)), ls='-', lw=2)
plt.xlabel('distance (mi)')
plt.ylabel('altitude (m)')
# plt.title('altitude')
plt.legend(loc='best')
plt.subplot(3, 1, 3)
plt.plot(distance/1600, speed)
plt.axis([0, distance.max()[0]/1600, speed.mean()[0]*0.9, speed.mean()[0]*1.1])
plt.axhline(y=speed.mean()[0], xmin=0, xmax=1, c='r', label = 'mean = ' + str(speed.mean()[0].round(2)), ls='-', lw=2)
plt.xlabel('distance (mi)')
plt.ylabel('speed (m/s)')
# plt.title('speed')
plt.legend(loc='best')
plt.tight_layout()
plt.show()
In [16]:
plt.figure(figsize=(11,3))
plt.scatter(locs.lat, locs.lon, c = cadence, s=800, vmin=86, vmax=92, marker='.', edgecolor='none', alpha=0.08, cmap=cm.hot_r)
plt.xlabel('Latitude')
plt.ylabel('Longitude')
plt.title('Long slow death by running (finish line is on the left)')
plt.colorbar().set_label('Cadence', labelpad=-20, y=1.1, rotation=0)
That projection sucks (its raw lat, lon), so use your fav mapper..¶
Gmaps¶
In [18]:
gmaps.configure(api_key="XXXXXXXXXXXXXXXXXXXXXXXXXXXX") # Your Google API key
In [19]:
fig = gmaps.figure() # zoom_level=2
data = gmaps.symbol_layer(locs, scale = 1)
fig.add_layer(data)
# fig
Calling fig here spits out a gmap of the route
Its zoomable, clickable, pan-able.. but its static html, and I don't want to upload that to GitHub, so here's a screenshot:
Try using Bokeh¶
In [17]:
from bokeh.models import GMapPlot, GMapOptions, ColumnDataSource, Circle, Range1d, PanTool, WheelZoomTool, ResetTool, SaveTool
from bokeh.io import show, output_notebook
from bokeh.embed import components
output_notebook()
In [29]:
# calculate middle lat and lon, for map centering
midlat = locs['lat'].min()+((locs['lat'].max()-locs['lat'].min())/2)
midlon = locs['lon'].min()+((locs['lon'].max()-locs['lon'].min())/2)
In [27]:
map_options = GMapOptions(lat=midlat, lng=midlon, map_type="roadmap", zoom=11)
activitymap = GMapPlot(x_range=Range1d(), y_range=Range1d(), map_options=map_options)
activitymap.api_key = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
source = ColumnDataSource(data=locs)
circle = Circle(x="lon", y="lat", size=3, fill_color="blue", fill_alpha=0.9, line_color=None)
activitymap.add_glyph(source, circle)
activitymap.add_tools(PanTool(), WheelZoomTool(), ResetTool())
In [28]:
show(activitymap)
Does the map show up above this prompt? Probably not..¶
That's a Pelican --> HTML problem. Works fine in Jupyter notebooks. Too lazy to fix it here, just use the web app instead