parsed-data.html

<!--
See extensive explanation in the mai readme.

Just to clarify, the workers do the following: 
    set - always sends header to main, cache uses fkey 
    cut - always sends header+data to main, cache uses fkey 
    pos - always sends header, xy, dir, and speed to main, cache uses object with fkey and post_processing options*
    tet - sends one or more of header, amps, times, gl_data. cache uses fkey
* note that obejcts' identities are not relevant, it's the value of their properties that counts

Note that in the worker we bump parsed data in the FIFO when it is re-requested, but only when the worker
hears about those re-requests.  Perhaps the main thread should send bump signals to the worker when it
uses something from cache.  Exactly what happens to these signals doesn't matter, they're just to improve
the utilization of the cqche.

TODO: angle correction for dir from set file.
-->


<link rel="import" href="bower_components/polymer/polymer.html">
<link rel="import" href="worker-builder.html">
<link rel="import" href="utils.html">

<dom-module id="parsed-data">

<script is='worker-builder' id="io_worker" title="io-worker" type='javascript/worker'>
"use strict";

var timer = 0;
var ports = {};
var pending_files = {};
var pending_fkeys = {};
var pending_options = {};
var types = ['tet', 'set', 'pos', 'cut']; // this is the priority order
var parsers_cached_fkeys = {
    tet: new Set(), // tracks which fkeys tet worker currently holds in its cache buffers
    pos: new Set() // same for pos
}
var max_cache_size = 4; // pretty arbitrary, especially as it's not type-specific
var currently_reading;

var got_ports = function(ports_){
    ports = ports_;
    for(let t of types){
        let t_local = t;
        ports[t].onmessage = function(e){
            if(e.data !== 'done reading'){
                throw "bad message from " + t + " to io worker: " + JSON.String(data);
            }
            if (currently_reading === t_local){
                currently_reading = undefined;
                touch_timer();
            }
        }
    }
}

var read_files = function(files, options){
    for(let t of types){
        if(files[t] !== undefined){
            pending_files[t] = files[t];
            pending_options[t] = options[t];
            pending_fkeys[t] = files[t+'_fkey'];
        }
    }
    touch_timer();
}

var touch_timer = function(){
    for(let t of types){
        if(pending_files[t]){
            timer = timer || setImmediate(timer_tick);
            return;
        }
    }
    clearImmediate(timer);
    timer = 0;
}

var timer_tick = function(){
    timer = 0;
    // if there are any files already cached, then issue those requests immediately
    for(let t of types){
        if(!pending_files[t] || !parsers_cached_fkeys[t]){
            continue;
        }
        let file = pending_files[t];
        let fkey = pending_fkeys[t];
        let cache = parsers_cached_fkeys[t];
        if(cache.has(fkey)){
            // bump in FIFO
            cache.delete(fkey); 
            cache.add(fkey); 
            ports[t].postMessage({
                fkey: fkey,
                name: file.name,
                file: null, // buffer is available at the reciever's end!!
                options: pending_options[t],
                delete_from_cache: null 
            });
            pending_files[t] = undefined;
            pending_fkeys[t] = undefined;
            pending_options[t] = undefined;
        }
    }

    if(currently_reading){
        return; // and do not touch_timer, instead wait for the message from the current reader or from main thread
    }

    // now, load zero or one files from disk and issue request to corresponding worker
    // crucially, we do this in the priority order defined by the types array
    for(let t of types){
        if(!pending_files[t]){
            continue;
        }
        let file = pending_files[t];
        let fkey = pending_fkeys[t];
        let cache = parsers_cached_fkeys[t];
        let delete_from_cache;
        if(cache){
            // note that fkey can't have been in the cache already, because we tested for that in the above loop.
            cache.add(fkey);
            if(cache.size > max_cache_size){
                delete_from_cache = cache.keys().next().value;
                cache.delete(delete_from_cache);
            }
        }
        currently_reading = t; // see note about Chrome bug at top of page
        //let reader = new FileReaderSync();  
        //let buf = reader.readAsArrayBuffer(file);
        ports[t].postMessage({
            fkey: fkey,
            name: file.name,
            file: file,
            options: pending_options[t],
            delete_from_cache: delete_from_cache
        });
        pending_files[t] = undefined;
        pending_options[t] = undefined;
        pending_fkeys[t] = undefined;
        break;
    }

    // and keep going...
    touch_timer();
}
</script>


<script is='worker-builder' id="pos_parser" title="pos-parser" type='javascript/worker'>
"use strict";

var fkey_to_buffer = new Map();
var main_cached_opts = new Set(); // set of objects each giving options and fkey
var max_cache_size = 4; // arbitrary

var got_ports = function(ports){
    self.io_port = ports.pos;
    io_port.onmessage = function(e){
        var fkey = e.data.fkey;
        var buffer = fkey_to_buffer.get(fkey);
        if(!buffer){
            // see note at top of page on Chrome bug
            let reader =  new FileReaderSync();  // see note about Chrome bug at top of page
            buffer = reader.readAsArrayBuffer(e.data.file);
        }
        io_port.postMessage('done reading');

         // add/delete entries in buffer cache, based on instructions from io-worker
        fkey_to_buffer.set(fkey, buffer);
        if(e.data.delete_from_cache){
            fkey_to_buffer.delete(e.data.delete_from_cache);
        }

        // check to see if main cache actually holds the requested data     
        var options = e.data.options;
        options.fkey = fkey;
        for(let cached_opts of main_cached_opts){
            if(is_equal_simple(cached_opts, options)){
                // bump in FIFO
                main_cached_opts.delete(cached_opts);
                main_cached_opts.add(cached_opts);
                return; // nothing to be done
            }
        }
        
        // pre-emptively add to the record of main cache and decide what, if anything, to remove
        main_cached_opts.add(Object.assign({}, options));
        var delete_from_cache;
        if(main_cached_opts.size > max_cache_size){
            delete_from_cache = main_cached_opts.keys().next().value;
            main_cached_opts.delete(delete_from_cache);
        }

        parse_pos_file(buffer, fkey, options, e.data.name, delete_from_cache);

    }
}


var is_equal_simple = function(a, b){
    // very simple....
    if((a && !b) || (!a && b)){
        return false;
    }
    for(let aa in a){
        if(b[aa] !== a[aa]){
            return false;
        }
    }
    for(let bb in b){
        if(b[bb] !== a[bb]){
            return false;
        }
    }
    return true;
}

var swap_16 = function (val) {
    return ((val & 0xFF) << 8) | ((val >> 8) & 0xFF);
}

var take_2_2_swapped = function(src, offset_bytes, stride_bytes){
    // takes 4 bytes total, swapping bytes 0 and 1, and swapping bytes 2 and 3.

    var dest = new Uint32Array( (0 | (src.length/stride_bytes)) - (0 | (offset_bytes/stride_bytes)));

    for(let p_src=offset_bytes, p_dest=0; p_src<src.length;p_src=p_src+stride_bytes,p_dest++){
        dest[p_dest] =   src[p_src+1]
                       |(src[p_src+0] << 8)
                       |(src[p_src+3] << 16)
                       |(src[p_src+2] << 24);
    }
    return dest;
}

var take = function(data,offset,stride){
    // takes every stride'th element from data, starting with the offset'th element
    
    var n = data.length/stride;
    var res = new data.constructor(n);
    
    for(let i=0,j=offset;i<n;i++,j+=stride){
        res[i] = data[j];
    }
    return res;
}

var times_in_place = function(src, factor, skip_val){
    for(let i=0;i<src.length;i++)if(src[i] != skip_val){
        src[i] *= factor;
    }
}

var replace_val_in_place = function(src, find, replace){
    for(let i=0;i<src.length;i++)if(src[i] == find){
        src[i] = replace;
    }
}

var sqr = function(a){return a*a;}

var clone = function(a){ 
    if(a.slice){
        return a.slice(0); //for basic arrays and pure arraybuffer
    }else{
        return new a.constructor(a);
    }
}

var minus = function(a, b, c){
    // we subtract b and c from alternate elemetns of a, inplace, nan16 is skipped
    for(let i=0;i<a.length;i++){
        a[i] -= a[i] == nan16? 0 : b;
        i++;
        a[i] -= a[i] == nan16? 0 : c;
    }
}

const regex_header_a = /((?:[\S\s](?!\r\ndata_start))*[\S\s])(\r\ndata_start)/
const regex_header_b = /(\S*) ([\S ]*)/g

const bytes_per_pos_sample = 4 + 2 + 2 + 2 + 2 + 2 + 2 + (2 + 2) ;//the last two uint16s are numpix1 and numpix2 repeated
const nan16 = -32768; //custom nan value, equal to minimum int16 value    
const pos_nan = 1023;

var parse_pos_file = function(buffer, fkey, options, file_name, delete_from_cache){
    var top_str = new TextDecoder('utf-8').decode(buffer.slice(0, 1204+1));
    var match = regex_header_a.exec(top_str);
    if(!match){
        throw 'did not find end of header in pos file.';
    }

    var data_start = match.index + match[0].length;
    var header = {
        file: file_name
    };
    var header_str = match[0];
    while (match = regex_header_b.exec(header_str)){
        header[match[1]] = match[2];
    }

    if (header.pos_format !== "t,x1,y1,x2,y2,numpix1,numpix2"){
        throw "uncregonised pos format used in pos file";
    }
    
    // apply overrides
    for (let k in options.header_override){
        if(header[k] !== undefined){
            header[k+"_original"] = header[k];
        }
        header[k] = header_override[k];
    }

    var data_len_bytes = parseInt(header.num_pos_samples)*bytes_per_pos_sample;

    var pos = post_process(buffer, data_start, data_len_bytes, header, options);
    delete options.fkey;

    exec_main_b('got_pos',{
        fkey: fkey,
        header: header,
        options: options,
        xy: 'xy',
        dir: 'dir',
        speed: 'speed',
        delete_from_cache: delete_from_cache,
    }, {
        xy: pos.xy,
        dir: get_dir(pos, header),
        speed: get_speed(pos, header)
    });

}



var interp_xy_sub = function(xy,x_a,y_a,x_b,y_b,i,n_nans){
    //interpolates from element i-1 back to i-n_nans, where element i is x_b,y_b and element i-n_nans-1 is x_a,x_b
    var dx = (x_b-x_a)/(n_nans+1);
    var dy = (y_b-y_a)/(n_nans+1);
    for(let j=0;j<n_nans;j++){
        xy[(i-n_nans + j)*2+0] = x_a + (j+1)*dx;
        xy[(i-n_nans + j)*2+1] = y_a + (j+1)*dy; 
    }
}

var interp_xy = function(xy,n_pos){
   /* 
    Interpolates linearly across nan blocks for single xy stream.
    Does it in place.

    TODO: verify that this does exactly what we want
   */

    // Find first (x,y) that is non-nan
    for(var start=0; start<n_pos; start++){
        let ix = start*2+0;
        let iy = start*2+1;
        if(xy[ix] != nan16 && xy[iy] !=nan16)
            break;
    }

    var ix = start*2+0;
    var iy = start*2+1; 
    var x_a = xy[ix];
    var y_a = xy[iy];
    var n_nans = start; //this will cause first non-nan to be copied back through all previous nan values
    for(var i=start;i<n_pos;i++){
        let ix = i*2+0;
        let iy = i*2+1; 
        let x_b = xy[ix];
        let y_b = xy[iy];
        if(x_b == nan16 || y_b == nan16){
            n_nans++;
        }else{
            if(n_nans) 
                interp_xy_sub(xy, x_a, y_a, x_b, y_b, i, n_nans)
            x_a = x_b;
            y_a = y_b;
            n_nans = 0;
        }
    }
    
    if(n_nans) //fill end-nan values with last non-nan val
        interp_xy_sub(xy, x_a, y_a, x_a, y_a, i, n_nans);

}

var smooth_1d_in_place = function(x, stride, k){
    //box car smoothing of length 2*k + 1
    // (If we pretend the stride=1) The first few values of x will be:
    //  x[0] = (x[0] + x[1] + ... + x[k])/(k+1)
    //  x[1] = (x[0] + x[1] + ... + x[k+1])/(k+2)
    //  ... and then we get to..
    //  x[b] = (x[b-k] + ... + x[b] + ... x[b+k])/(2*k+1)
    // and then we ramp down at the end as with the start.
             
    //a couple of checks for unimplemented generalisations...
    if(stride != 2)
        throw("stride must be 2");
    if(2*k+1 > 256)
        throw("smoothing kernel max length is 256")
    if(k==0)
        return; //no smoothing

    /* Note: (a & 0xff) is (a mod 256) */
    var n = x.length/2;
    
    var circ_buff_1 = new x.constructor(256);
    var circ_buff_2 = new x.constructor(256);
    var tot_1 = 0;
    var tot_2 = 0;
    
    //a is the lowest-index in the sum, b is the central and destination index, c is the highgest index in the sum
    var a=-2*k,b=-k,c=0; 
    
    // ramp up part 1: push the first k values into the buffer and sum
    for(;c<k;a++,b++,c++){
        tot_1 += circ_buff_1[c & 0xff] = x[c*2 + 0]; 
        tot_2 += circ_buff_2[c & 0xff] = x[c*2 + 1];
    }
    
    // ramp up part 2: calculate the first k values
    for(;a<0;a++,b++,c++){
        tot_1 += circ_buff_1[c & 0xff] = x[c*2 + 0]; 
        tot_2 += circ_buff_2[c & 0xff] = x[c*2 + 1];
        x[b*2+0] = tot_1 / (c+1);
        x[b*2+1] = tot_2 / (c+1);
    }
        
    // main section
    var d = 2*k+1;
    for(;c<n;a++,b++,c++){
        tot_1 += circ_buff_1[c & 0xff] = x[c*2 + 0]; 
        tot_2 += circ_buff_2[c & 0xff] = x[c*2 + 1];
        x[b*2+0] = tot_1/d;
        x[b*2+1] = tot_2/d;
        tot_1 -= circ_buff_1[a & 0xff]; 
        tot_2 -= circ_buff_2[a & 0xff];
    }
    
    // ramp down: calculate last k values
    for(;b<n;a++,b++,c++){
        x[b*2+0] = tot_1 / (n-a);
        x[b*2+1 ] = tot_2 / (n-a);
        tot_1 -= circ_buff_1[a & 0xff]; 
        tot_2 -= circ_buff_2[a & 0xff];
    }
    
}

var jump_filter = function(xy, n_pos, max_speed, units_per_m, pos_timebase){
    /*
        For a single stream of xy values, it finds the first non-nan point,
        and then checks the speed required to reach the next point, given the
        sampling rate.  If the speed is too high, it skips that point and calcualtes
        the speed required to get to the following point. This continues until,
        the speed requirement is satisfied. The "skipped" points are set to NaN, inplace.
        The number of points skipped over is returned as an integer.
    */

    if(!max_speed)
        return 0;

    var sqr_max_samp_step = sqr(max_speed*units_per_m /pos_timebase);

    // Find first (x,y) that is non-nan
    for(var start=0; start<n_pos; start++){
        let ix = start*2+0;
        let iy = start*2+1;
        if(xy[ix] != nan16 && xy[iy] !=nan16)
            break;
    }
            
    var ix = start*2+0;
    var iy = start*2+1;     
    var x_from = xy[ix];
    var y_from = xy[iy];
    var jump_len = 1;

    // Set big jump sections to nan
    var n_jumpy = 0;
    for(let i=start+1; i<n_pos; i++){
        let ix = i*2+0;
        let iy = i*2+1;
        // check if this pos is already nan
        // or if (dx^2 + dy^2)/dt^2 is greater than maxSpeed^2, where the d's are relative to the last "good" sample
        if(xy[ix] == nan16 || xy[iy] == nan16){
            xy[ix] = xy[iy] = nan16; //just in case only one or the other was nan
            jump_len++; // note we don't count n_jumpy here
        }else if (sqr(x_from-xy[ix]) + sqr(y_from-xy[iy]) > sqr_max_samp_step *  sqr(jump_len)){
            //sample is nan or speed is too large, so make this a jump
            xy[ix] = xy[iy] = nan16; 
            n_jumpy++;
            jump_len++;
        }else{
            //speed is sufficiently small, so this point is ok
            jump_len = 1;
            x_from = xy[ix];
            y_from = xy[iy];
        }
    }

    return n_jumpy;
}       

var swap = function(a, b, do_swap){
    var n = 0;
    for(let i=0;i<a.length;i++) if(do_swap[i]){
        let tmp = a[i];
        a[i] = b[i];
        b[i] = tmp;
        n++;
    }
    return n;
}

var nanmean_and_std_2 = function(x){
    /* 
        x is nx2 array, we want nancount, nanmean, and nanstd for both columns.
    */

    var sum_1 = 0; var sum_2 = 0;
    var n1 = 0; var n2 = 0;

    for(let i=0; i<x.length/2; i++){
        let i1 = i*2+0;
        let i2 = i*2+1;
        if(x[i1] && x[i1] != nan16){
            n1++;
            sum_1 += x[i1];
        }
        if(x[i2] && x[i2] != nan16){
            n2++;
            sum_2 += x[i2];
        }
    }
    var mean_1 = sum_1/n1; var mean_2 = sum_2/n2;

    // now get sum(sqr(xy-mean_xy)) and use to calculate nanstd...
    sum_1 = 0; sum_2 = 0;  // NOTE: reusing sums vars!!!!
    for (let i=0; i<x.length/2; i++){
        let i1 = i*2+0;
        let i2 = i*2+1;
        if(x[i1] && x[i1] != nan16)
            sum_1 += sqr(x[i1] - mean_1);
        if(x[i2] && x[i2] != nan16)
            sum_2 += sqr(x[i2] - mean_2);
    }
    var std_1 = Math.sqrt(sum_1/n1); var std_2 = Math.sqrt(sum_2/n2);

    return {mean_1: mean_1, mean_2: mean_2, std_1: std_1, std_2: std_2, n_1: n1, n_2: n2};
}

var combine_xy = function(xy_1, xy_2, weight_1, weight_2){
    /*
        xy_1 and xy_2 are both streams of (x,y) values. 
        We combine them into a single stream according to the ratio of the weights.
    */
    var weight_sum = (weight_1 + weight_2);
    var weight_1 = weight_1/weight_sum;
    var weight_2 = weight_2/weight_sum;
    var ret = new xy_1.constructor(xy_1.length);
    for(let i=0;i<xy_1.length/2;i++){
        let ix = 2*i+0;
        let iy = 2*i+1;
        ret[ix] = xy_1[ix]*weight_1 + xy_1[ix]*weight_2;
        ret[iy] = xy_1[iy]*weight_1 + xy_1[iy]*weight_2;
    }
    return ret;
}

var get_dir = function(pos, header){
    let xy_1 = pos.xy_1;
    let xy_2 = pos.xy_2;
    let ret = new Float32Array(xy_1.length/2);
    let pi = 3.14159265;

    if(xy_2){
        let correction = (parseInt(header.bearing_colour_1)/180*pi) || 0; 
        for(let i=0;i<xy_1.length/2;i++){
            let ix = 2*i+0;
            let iy = 2*i+1;
            let dy = xy_2[iy] - xy_1[iy];
            let dx = xy_2[ix] - xy_1[ix];
            ret[i] = Math.abs(dx) < 0.001 && Math.abs(dy) < 0.001 ? NaN : (Math.atan2(dy, dx) + 2*pi + correction) % (2*pi);
        }        
    } else {
        // only one led used...
        for(let i=1;i<xy_1.length/2;i++){
            let dy = xy_1[2*i+1] - xy_1[2*i-1];
            let dx = xy_1[2*i+0] - xy_1[2*i-2];
            ret[i] = Math.abs(dx) < 0.001 && Math.abs(dy) < 0.001 ? NaN : (Math.atan2(dy, dx) + 2*pi) % (2*pi);
        }
        ret[0] = ret[1];
    }
    return ret;
}

var get_speed = function(pos, header){
    var xy = pos.xy;
    var ret = new Float32Array(xy.length/2);
    let f =  parseInt(header.sample_rate)/header.units_per_meter*100;
    for(let ii=0; ii<xy.length; ii=ii+2){
         ret[ii>>1] = Math.hypot(xy[ii+2]-xy[ii+0], xy[ii+3]-xy[ii+1])*f;
    }
    return ret;
}


var any_nonzero = function(x, offset, stride){
    for(let ii=offset; ii<x.length; ii+= stride){
        if(x[ii] && x[ii] != pos_nan){
            return true;
        }
    } 
    return false;
}

var compute_zscores = function(xy_pix, n_pos, mean_1, mean_2, std_1, std_2){
    var dest = new Uint8Array(n_pos);  

    for(let i=0; i< n_pos; i++){
        let i1 = i*2+0;
        let i2 = i*2+1;
        if(xy_pix[i1] && xy_pix[i2] && xy_pix[i1] != nan16 && xy_pix[i2] != nan16){
            let z11 = (mean_1 - xy_pix[i1])/std_1;
            let z12 = (xy_pix[i1] - mean_2)/std_2;
            dest[i] = z11 > z12;
        }
    }
    return dest;
}

var use_zscore = function(xy_1, xy_2, n_pos, shrunk_and_switched, swapping_thresh){
    // Find first (x,y) that is non-nan on both xy and xy_2
    let start;
    for(start=0; start<n_pos; start++){
        let ix = start*2+0;
        let iy = start*2+1;
        if(xy_1[ix] != nan16 && xy_1[iy] !=nan16 && xy_2[ix] != nan16 && xy_2[iy] !=nan16)
            break;
    }

    for(let i = start+1; i<n_pos; i++)if(shrunk_and_switched[i]){
        // we are going to do diffs with XY_i - XY_(i-1)
        let ix = i*2+0;
        let iy = i*2+1;
        let i_1x = i*2-2;
        let i_1y = i*2-1;
        if(xy_1[ix] == nan16 || xy_1[iy] == nan16 || xy_2[ix] == nan16 || xy_2[iy] == nan16){
            shrunk_and_switched[i] = 0;
            i++; // skip next iteration as well becuase the current index cannot be used as i, or as (i-1)
            shrunk_and_switched[i] = 0;
            continue; 
        }

        let dist12 = Math.hypot(xy_1[ix] - xy_2[i_1x],  xy_1[iy] - xy_2[i_1y]);
        let dist11 = Math.hypot(xy_1[ix] - xy_1[i_1x],  xy_1[iy] - xy_1[i_1y]);
        let dist21 = Math.hypot(xy_2[ix] - xy_1[i_1x],  xy_2[iy] - xy_1[i_1y]);
        let dist22 = Math.hypot(xy_2[ix] - xy_2[i_1x],  xy_2[iy] - xy_2[i_1y]);

        shrunk_and_switched[i] = (dist12 < dist11-swapping_thresh) && (dist21 < dist22 - swapping_thresh);
    }
}

var post_process = function(buffer, offset_bytes, length_bytes, header, options){
    let data8 = new Uint8Array(buffer, offset_bytes, length_bytes);

    let elements_per_pos_sample = bytes_per_pos_sample/2;
    let n_pos = parseInt(header.num_pos_samples); 
    let end = n_pos * elements_per_pos_sample; 
    
     // for each pos sample take bytes 4-7, and then view them as a pair of int16s 
    let xy_1 = new Int16Array(take_2_2_swapped(data8, 4, bytes_per_pos_sample).buffer);
    replace_val_in_place(xy_1, pos_nan,nan16); //switch from axona custom nan value to our custom nan value
    
    let n_led = 1;
    let xy_2, xy_pix;
    if (options.use_both_leds){
        // Rather than using colactive header value in set file (which is a massive pain to get asynchrously here)
        // We see if any of the pixel counts are non-zero/non-nan for the second led, to establish how many leds were used.
        xy_pix = new Int16Array(take_2_2_swapped(data8, 12, bytes_per_pos_sample).buffer);
        replace_val_in_place(xy_pix, pos_nan, nan16); //switch from axona custom nan value to our custom nan value
        n_led = any_nonzero(xy_pix, 1, 2) ? 2 : 1;

        if(n_led == 2){
             // for each pos sample take bytes 8-11, and then view them as a pair of int16s 
            xy_2 = new Int16Array(take_2_2_swapped(data8, 8, bytes_per_pos_sample).buffer);
            replace_val_in_place(xy_2, pos_nan, nan16); //switch from axona custom nan value to our custom nan value
        }
    }
    
    if(options.need_to_subtract_mins){
        var min_x = parseInt(header.window_min_x);
        var min_y = parseInt(header.window_min_y);
        minus(xy_1, min_x, min_y);
        if(n_led == 2)
            minus(xy_2, min_x, min_y);
    }

    let ppm = parseInt(header.pixels_per_metre);
    const units_per_m = 10000;
    times_in_place(xy_1, units_per_m/ppm, nan16); //convert from pixels to milimeters (we use mm because then we can happily use Int16s)
    if(n_led == 2){
        times_in_place(xy_2, units_per_m/ppm, nan16); //convert from pixels to milimeters (we use mm because then we can happily use Int16s)
    }

    let weight_1, weight_2, mean_1, mean_2, std_1, std_2;
    if(n_led == 2){
        // check for and apply LED swaping...
        let swapping_thresh = options.swapping_thresh_cm * (units_per_m/100);

        // firstly we check to see if number of pixels for first LED is actually closer to pixel count mean for second led,
        // where "closer" is defined as z-score, i.e. distance/std for the relevant distribution.
        let pix_props = nanmean_and_std_2(xy_pix);
        weight_1 = pix_props.n_1/n_pos; weight_2 = pix_props.n_2/n_pos;

        mean_1 = pix_props.mean_1; mean_2 = pix_props.mean_2; std_1 = pix_props.std_1; std_2 = pix_props.std_2;
        header.post_n_pix_led1 = "mean=" + pix_props.mean_1.toFixed(2) + " std=" + pix_props.std_1.toFixed(2) + " (nan count=" + (n_pos - pix_props.n_1) + ")";
        header.post_n_pix_led2 = "mean=" + pix_props.mean_2.toFixed(2) + " std=" + pix_props.std_2.toFixed(2) + " (nan count=" + (n_pos - pix_props.n_2) + ")";

        // use std and mean to get z score of pix1 to pix1 and pix2
        let shrunk_and_switched = compute_zscores(xy_pix, n_pos, mean_1, mean_2, std_1, std_2);

        // Now we calculate jump distance (from time i-1 to time i)
        // four distnaces: led1 to led1, led1 to led2, led2 to led1, led2 to led2.
        // if the recorded version of distance is more than swapping_thresh further
        // than the potential "swapped" version, then consider it a swap.
        use_zscore(xy_1, xy_2, n_pos, shrunk_and_switched, swapping_thresh)

        // Swap xy_1 with xy_2 where we decided we need to swap. (Note we use 32bit to swap 2x16bit xy in one go)
        header.post_n_swapped = swap(new Uint32Array(xy_1.buffer), new Uint32Array(xy_2.buffer), shrunk_and_switched);
    }
    
    let pos_timebase = parseInt(header.sample_rate);
    header.post_n_jumpy_led1 = jump_filter(xy_1, n_pos, options.max_speed, units_per_m, pos_timebase)
    if(n_led == 2){
         header.post_n_jumpy_led2 = jump_filter(xy_2, n_pos, options.max_speed, units_per_m, pos_timebase);
    }
    
    interp_xy(xy_1,n_pos);
    if(n_led == 2){
        interp_xy(xy_2,n_pos);
    }

    let k = Math.floor(pos_timebase*options.smoothing_secs/2); //the actual filter will be of length k*2+1, which means it may be one sample longer than desired         
    smooth_1d_in_place(xy_1, 2, k);
    let xy;
    if(n_led == 2){
        smooth_1d_in_place(xy_2, 2, k);
        xy = combine_xy(xy_1, xy_2, weight_1, weight_2);
    }else{
        xy = xy_1;
    }

    header.max_vals = [(parseInt(header.window_max_y)-parseInt(header.window_min_y))*units_per_m/ppm ,
                       (parseInt(header.window_max_x)-parseInt(header.window_min_x))*units_per_m/ppm ]; //TODO: decide which way round we want x and y
    header.units_per_meter = units_per_m;

    return {
        xy: xy,
        xy_1: xy_1,
        xy_2: xy_2,
        header: header
    }
}

</script>


<script is='worker-builder' id="tet_parser" title="tet-parser" type='javascript/worker'>
"use strict";
// TODO: it might be nice, though complicated, to batch the tet parsing, that would
// allow the io to overlap with the compute, both of which are about 1ms/1k spikes.

var fkey_to_buffer = new Map();

var main_cached_opts = new Set();
var max_cache_size = 4; //aribtrary

var got_ports = function(ports){
    self.io_port = ports.tet;
    io_port.onmessage = function(e){
        var fkey = e.data.fkey;
        var buffer = fkey_to_buffer.get(fkey);
        if(!buffer){
            // see note at top of page on Chrome bug
            let reader =  new FileReaderSync();  // see note about Chrome bug at top of page
            buffer = reader.readAsArrayBuffer(e.data.file);
        }
        io_port.postMessage('done reading');

        // add/delete entries in buffer cache, based on instructions from io-worker
        fkey_to_buffer.set(fkey, buffer);
        if(e.data.delete_from_cache){
            fkey_to_buffer.delete(e.data.delete_from_cache);
        }

        // check to see if main cache actually holds any of the requested data 
        var options = e.data.options;       
        options.fkey = fkey;
        let cached_opts;
        for(cached_opts of main_cached_opts){
            if(cached_opts.fkey === fkey){
                // bump in FIFO
                main_cached_opts.delete(cached_opts);
                main_cached_opts.add(cached_opts);
                // remove the things that are already in cache
                if(cached_opts.times){
                    delete options.times;
                }
                if(cached_opts.amps){
                    delete options.amps;
                }
                if(cached_opts.gl_data){
                    delete options.gl_data;
                }
                if(options.times || options.amps || options.gl_data){
                    break; // there is at least some work to be done
                } else{
                    return; // nothing new requested
                }
            }
        }   

        if (!cached_opts || cached_opts.fkey !== fkey){
            // create new entry in main cache record, and possibly delete something
            cached_opts = {fkey: fkey};
            main_cached_opts.add(cached_opts);
            if(main_cached_opts.size > max_cache_size){
                let delete_from_cache = main_cached_opts.keys().next().value;
                exec_main('delete_from_tet_cache', delete_from_cache.fkey);
                main_cached_opts.delete(delete_from_cache);
            }
        }

        // pre-emptively record what we are about to put in main cache
        if(options.times){
            cached_opts.times = 1;
        }
        if(options.gl_data){
            cached_opts.gl_data = 1;
        }
        if(options.amps){
            cached_opts.amps = 1;
        }

        // and actually do the work...
        parse_tet_file(buffer, fkey, options, e.data.name);

    }
}

const regex_header_a = /((?:[\S\s](?!\r\ndata_start))*[\S\s])(\r\ndata_start)/
const regex_header_b = /(\S*) ([\S ]*)/g
const data_end = "\r\ndata_end";
const n_c = 4;
const n_w = 50;
const bytes_per_spike = n_c*(4 + n_w);

var parse_tet_file = function(buffer, fkey, options, file_name){
    var top_str = new TextDecoder('utf-8').decode(buffer.slice(0, 1204+1));
        
    var match = regex_header_a.exec(top_str);
    if(!match){
        throw 'did not find end of header in tet file.';
    }

    var data_start = match.index + match[0].length;
    var header = {
        file: file_name
    };
    var header_str = match[0];
    while (match = regex_header_b.exec(header_str)){
        header[match[1]] = match[2];
    }

    if (header.spike_format !== "t,ch1,t,ch2,t,ch3,t,ch4"){
        throw "uncregonised spike format used in tet file";
    }
    
    // sometimes DaCQ creates a header with num_spikes >0, but there are no spikes
    // (this happens when you choose not to record a given tetrode but a file previously existed)
    if(top_str.slice(data_start, data_start + data_end.length) === data_end){
        header.num_spikes_claimed = header.num_spikes;
        header.num_spikes = 0;
    }
    var n = parseInt(header.num_spikes);
    var data_len_bytes = n*bytes_per_spike;

    exec_main('got_tet_header', fkey, header);

    // TODO: test whether it's faster to iterate over the data once and compute everything
    // or to iterate once for each thing of interest (which is a bit more complicated)
    // if we stick with the multiple separate iterations, then decide on a good priority order
    // that gives users the best perception of speed...gl data probably comes first as it is
    // worth having even without cut data, whereas the other two are less helpful without cut
    // data.

    if(options.gl_data){
        var data =  build_gl_data(buffer, data_start, data_len_bytes, n);
        exec_main_b('got_gl_data', {
            data: 'data', 
            fkey: fkey,
            n_spikes: n
        }, {data: data});
    }

    if(options.times){
        var data =  get_times(buffer, data_start, data_len_bytes, n);
        exec_main_b('got_times', {
            data: 'data', 
            fkey: fkey,
            n_spikes: n,
            timebase: parseInt(header.timebase),
        }, {data: data});
    }

    if(options.amps){
        var data = get_amplitudes(buffer, data_start, data_len_bytes, n);
        var good_chans = check_channels(data);
        exec_main_b('got_amplitudes', {
            data: 'data', 
            chan_is_ok: good_chans,
            fkey: fkey,
            n_spikes: n
        }, {data: data});
    }
}

var build_gl_data_sub = function(data_in, data_out_16, n_spikes){
    // Note how we read from data_in contiguously, but write out non-contiguously.
    // This is about 4x faster than doing it the other way around.
    // It takes about 80ms for 80k spikes.
    // DataView should allow for fast misaligned uint16 access of data_in, but currently it's slow...
    // https://bugs.chromium.org/p/chromium/issues/detail?id=225811. Even if it's optimized in chrome
    // it probably won't help by more than 5-10% I would think.

    var q = -1;
    for(let i=0, p=0;i<n_spikes;i++,p=i){ //for each spike
        for(let c=0;c<n_c;c++){ //for each channel
            q = q + 5;
            for(let t=0;t<n_w-1;t++){ //for each time point (except the last one)
                data_out_16[p] = data_in[q] | (data_in[++q] << 8); // TODO: deal properly with endianness of system 
                // (Note that even though we are drawing a stand alone line segment from a to b, we still need to know how they match up to times t and t+1)
                p = p + n_spikes;
            }
        }
    }
}

var build_gl_data = function(buffer, offset_bytes, length_bytes, n_spikes){
    var old_data = new Uint8Array(buffer, offset_bytes, length_bytes);
    var new_data = new Uint16Array(n_c*(n_w-1)*n_spikes);
    
    build_gl_data_sub(old_data, new_data, n_spikes);
    return new_data;        
}


var get_times = function(buffer, offset_bytes, length_bytes, n_spikes){ //get spike times in milliseconds as a Uint32Array 
    var times = new Uint32Array(n_spikes);
    var data8 = new Uint8Array(buffer, offset_bytes, length_bytes); // can't make this int32 because it may not be sufficeintly alligned

    if (endian === 'L') {
        for(let i=0, p=0; i<times.length; i++, p+= bytes_per_spike){
            times[i] = (data8[p+0] << 24)
                     | (data8[p+1] << 16)   
                     | (data8[p+2] << 8) 
                     | (data8[p+3] << 0); 
        }        
    } else {
        throw "need to implement for big endian"; // TODO: or maybe the version above is actually valid for bothered?
    }

    return times;
}

var get_amplitudes_sub = function(old_data, amps, n_sc, n_w){
    n_sc = n_sc | 0; // int
    n_w = n_w | 0; // int
    for(let i=0, p=0; i<n_sc; i++){
        p = p+ 4; // skip timestamp 
        let min = 127;
        let max = -128;
        for(let t=0; t<n_w; t++,p++){
            (old_data[p] > max) && (max = old_data[p]);
            (old_data[p] < min) && (min = old_data[p]);
        }
        amps[i] = max-min; 
    }
}

var check_channels = function(amps){
    let small_amp = 5; // arbitrary threshold

    let chan_is_ok = new Uint8Array(n_c);
    for(let c=0; c<n_c; c++){
        for(let ii=c; ii<amps.length; ii+=n_c){
            chan_is_ok[c] = (amps[ii] > small_amp);
            if(chan_is_ok[c]){
                break;
            }
        }
    }
    return chan_is_ok;
}

var get_amplitudes = function(buffer, offset_bytes, length_bytes, n_spikes){
    var old_data = new Int8Array(buffer, offset_bytes, length_bytes);
    var amps = new Uint8Array(n_spikes * n_c);
    get_amplitudes_sub(old_data, amps, n_spikes * n_c, n_w);

    return amps;
}

</script>


<script is='worker-builder' id="set_parser" title="set-parser" type='javascript/worker'>
"use strict";

var max_cache_size = 4; // arbitrary
var main_cached_fkeys = new Set();
var fkey_to_buffer = new Map();

var got_ports = function(ports){
    self.io_port = ports.set;
    io_port.onmessage = function(e){
        var fkey = e.data.fkey;
        var buffer = fkey_to_buffer.get(fkey);
        if(!buffer){
            // see note at top of page on Chrome bug
            let reader =  new FileReaderSync();  // see note about Chrome bug at top of page
            buffer = reader.readAsArrayBuffer(e.data.file);
        }
        io_port.postMessage('done reading');

        // add/delete entries in buffer cache, based on instructions from io-worker
        fkey_to_buffer.set(fkey, buffer);
        if(e.data.delete_from_cache){
            fkey_to_buffer.delete(e.data.delete_from_cache);
        }

        if(main_cached_fkeys.has(fkey)){
            // bump in FIFO
            main_cached_fkeys.delete(fkey);
            main_cached_fkeys.add(fkey);
            return;
        }
        var full_str = new TextDecoder('utf-8').decode(buffer);
        
        main_cached_fkeys.add(fkey);
        var delete_from_cache;
        if(main_cached_fkeys.size > max_cache_size){
            delete_from_cache = main_cached_fkeys.keys().next().value;
            main_cached_fkeys.delete(delete_from_cache);
        }

        var header = parse_set_file(full_str, fkey);
        header.file = e.data.name;
        exec_main('set_file_parsed', {
            fkey: fkey,
            header: header,
            delete_from_cache: delete_from_cache});

    }
}

const regex_header_b = /(\S*) ([\S ]*)/g

var parse_set_file = function(full_str, fkey){
    var header = {};
    var match;
    while (match = regex_header_b.exec(full_str))
        header[match[1]] = match[2];
    return header;
}

</script>


<script is='worker-builder' id="cut_parser" title="cut-parser" type='javascript/worker'>
"use strict";

const max_cache_size = 4; // arbitrary
const main_cached_fkeys = new Set();
const regex_ext = /\.([0-9a-z]+)$/i;
const fkey_to_buffer = new Map();

var got_ports = function(ports){
    self.io_port = ports.cut;
    io_port.onmessage = function(e){
        var fkey = e.data.fkey;
        var buffer = fkey_to_buffer.get(fkey);
        if(!buffer){
            // see note at top of page on Chrome bug
            let reader =  new FileReaderSync();  // see note about Chrome bug at top of page
            buffer = reader.readAsArrayBuffer(e.data.file);
        }
        io_port.postMessage('done reading');

        // add/delete entries in buffer cache, based on instructions from io-worker
        fkey_to_buffer.set(fkey, buffer);
        if(e.data.delete_from_cache){
            fkey_to_buffer.delete(e.data.delete_from_cache);
        }

        if(main_cached_fkeys.has(fkey)){
            // bump in FIFO
            main_cached_fkeys.delete(fkey);
            main_cached_fkeys.add(fkey);
            return; // main already has it, so no need to send it.
        }
        var full_str = new TextDecoder('utf-8').decode(buffer);

        // TODO: clu/cut based on extension
        var ext = e.data.name.match(regex_ext)[1]
        var msg;
        if (ext === 'cut') {
            msg = parse_cut_file(full_str, fkey);
        } else if(!isNaN(parseInt(ext))){
            msg = parse_clu_file(full_str, fkey);
        } else{
            throw "urecognised cut type";
        }
        
        var cut = msg.cut;
        msg.cut = "cut";
        msg.fkey = fkey;
        msg.header.file = e.data.name;
        main_cached_fkeys.add(fkey);
        if(main_cached_fkeys.size > max_cache_size){
            msg.delete_from_cache = main_cached_fkeys.keys().next().value;
            main_cached_fkeys.delete(msg.delete_from_cache);
        }

        exec_main_b('cut_file_parsed', msg, {cut : cut});

    }
}

const regex_cut_a = /n_clusters:\s*(\S*)\s*n_channels:\s*(\S*)\s*n_params:\s*(\S*)\s*times_used_in_Vt:\s*(\S*)\s*(\S*)\s*(\S*)\s*(\S*)/;
const regex_cut_b = /Exact_cut_for: ((?:[\s\S](?! spikes:))*[\s\S])\s*spikes: ([0-9]*)/;
const regex_cut_c = /([0-9]+)/g;
const max_length_match_cut_b = 300;//this is needed so that when we read in chunks of the cut fkey we dont have to apply the regex_b to the whole thing each time

var parse_cut_file = function(full_str, fkey){
    var match = regex_cut_a.exec(full_str);      
    var cut_props = {};
    cut_props.n_clusters =  parseInt(match[1]);
    cut_props.n_channels =  parseInt(match[2]);
    cut_props.n_params =  parseInt(match[3]);

    match = regex_cut_b.exec(full_str);
    cut_props.exp = match[1];
    cut_props.n_spikes = parseInt(match[2]);
    cut_props.data_start = match.index + match[0].length;
    cut_props.is_clu = false;

    var cut_as_str = full_str.slice(cut_props.data_start).match(regex_cut_c);
    var cut = new Uint32Array(cut_as_str.length);
    for(let ii=0; ii<cut.length; ii++){
        cut[ii] = parseInt(cut_as_str[ii]);
    }

    return {
        header: cut_props,
        cut: cut
    }

}

var parse_clu_file = function(full_str, fkey){
    var cut_as_str = full_str.match(regex_cut_c);
    var cut_props = {
        n_clusters: parseInt(cut_as_str[0]),  // TODO: check +/- 1
        n_spikes: cut_as_str.length-1,
        is_clu: true
    }
    var cut = new Uint32Array(cut_as_str.length-1);
    for(let ii=0; ii<cut.length; ii++){
        cut[ii] = parseInt(cut_as_str[ii+1]) -1;
    }

    return {
        header: cut_props,
        cut: cut
    }
}

</script>

<template></template>


<script>
    "use strict";
    Polymer({
        is:'parsed-data',
        properties: {
            selected_files: {
                type: Object,
                value: function(){return {set: null, pos: null, tet: null, cut: null}}, //fkeys
                notify: true,
                observer: '_change_to_files'
            },
            use_n_leds: {
                type: Number, //1 or 2
                value: 2,
                notify: true
            },
            speed_filter_mps: {
                type: Number,  //mps= meters per second
                value: 4,
                notify: true
            },
            pos_smoothing_s: {
                type: Number, 
                value: 0.2,
                notify: true
            },
            duration: {
                type: Number, // seconds
                value: 0,
                notify: true,
                readOnly: true
            },
            start_end_times: {
                type: Object, // start and end miliseconds (numbers)
                value: null,
                notify: true,
                readOnly: true
            },
            spike_times: {
                type: Object,
                value: function(){ return {};}, // data (akey), timebase
                readOnly: true,
                notify: true
            },
            pos_xy: {
                type: Object,
                value: function(){ return {};}, // xy (akey), timebase,  and units_per_cm
                readOnly: true,
                notify: true 
            },
            pos_dir: {
                type: Object,
                value: function(){ return {};}, // dir (akey), timebase
                readOnly: true,
                notify: true
            },
            pos_speed: {
                type: Object,
                value: function(){ return {};}, // speed (akey), timebase
                readOnly: true,
                notify: true
            },
            pos_header: {
                type: Object,
                value: function(){ return {};},
                readOnly: true,
                notify: true
            },
            set_header: {
                type: Object,
                value: function(){ return {};},
                readOnly: true,
                notify: true
            },
            tet_header: {
                type: Object,
                value: function(){ return {};},
                readOnly: true,
                notify: true
            },
            cut_box: {
                type: Object, // is: ["stack" | "file" | "n" | <undefined> ]
                              // is="file". akey: indices for file, header: header for file
                              // is="n". n_spikes in tet file
                              // is="stack". see cut-obj for details
                value: function(){ return {} },
                readOnly: true,
                notify: true
            },
            amplitudes: {
                type: Object,
                value: function(){return {};}, // n, chan_is_ok bool[4], amps taid
                notify: true,
                readOnly: true
            },
            data_for_gl: {
                type: Object,
                value: function(){return {};}, // has n value and typed-array-manager id for gl voltages
                readOnly: true,
                notify: true
            },
            want_spike_times: {
                type: Array,
                value: function(){return [];},
                notify: true
            },
            want_amplitudes: {
                type: Array,
                value: function(){return [];},
                notify: true
            },
            want_data_for_gl: {
                type: Array,
                value: function(){return [];},
                notify: true
            },
            no_groups: { // unlike everything else, this is only updated in the second phase of a file-switch
                type: Boolean,
                value: true,
                notify: true,
                readOnly: true
            }
        }, observers: [
            '_pos_options_changed(speed_filter_mps, use_n_leds, pos_smoothing_s)',
            '_tet_options_changed(want_spike_times.splices, want_data_for_gl.splices, want_amplitudes.splices)'
        ], created: function(){
            this._files = {}; // set, tet, pos, cut
            this._fkey_ids = {}; // set, tet, pos, cut
            this._cut_from_fkey = new Map();
            this._set_from_fkey = new Map();
            this._pos_from_options = new Map(); // key=options object giving fkey and parse options; value= {header, xy, dir, speed}
            this._tet_from_fkey = new Map(); // contains objects with one or more of {header, amps, times, gl_data}

            // this is slightly gratuitous as the set worker and cut workers really don't do all that much
            // but it's unlikely to be worse having them as workers.
            this._worker_io = Polymer.DomModule.import('parsed-data', '#io_worker').create_for(this);
            this._worker_set = Polymer.DomModule.import('parsed-data', '#set_parser').create_for(this);
            this._worker_tet = Polymer.DomModule.import('parsed-data', '#tet_parser').create_for(this);
            this._worker_pos = Polymer.DomModule.import('parsed-data', '#pos_parser').create_for(this);
            this._worker_cut = Polymer.DomModule.import('parsed-data', '#cut_parser').create_for(this);
            var ports_for_io = {};

            var c = new MessageChannel();
            ports_for_io['set'] = c.port1;
            this._worker_set.send_ports({set: c.port2});

            c = new MessageChannel();
            ports_for_io['tet'] = c.port1;
            this._worker_tet.send_ports({tet: c.port2});

            c = new MessageChannel();
            ports_for_io['pos'] = c.port1;
            this._worker_pos.send_ports({pos: c.port2});

            c = new MessageChannel();
            ports_for_io['cut'] = c.port1;
            this._worker_cut.send_ports({cut: c.port2});

            this._worker_io.send_ports(ports_for_io);

        }, attached: function(){
            var els = document.getElementsByTagName('cut-object');
            for(let ii=0; ii<els.length; ii++){
                this.listen(els[ii], 'fork', '_cut_forked');
            }
        }, _cut_forked: function(e){
            // We catch this signal from the cut-obj directly, in addition to the file-organiser catching it.
            // By doing this, when the file-organiser changes its file_selection, we will not treat the forked cut
            // as being new..which is great.  Note that setting the cut_box here will send it to the cut obj, but the
            // cut object will find that each inidvidual property is the same as before, so will not actually do anything.
            // Another thing to note is that file_selection is updated from within a debounce in file-organiser, which ensures
            // that this event handler is called before the file-organiser notifies us of the change.
            this._fkey_ids.cut = null;
            this._files.cut = e.detail;
            this._setCut_box(this._files.cut);
        }, _change_to_files: function(new_fkeys){
            // nullify things that are changing... (note we use this._fkey_ids rather than 2nd argument, old_fkeys due to _cut_forked)
            var changing = new Set();
            for(let k of ['tet','cut','pos','set']){
                if(!this._files[k] || !new_fkeys[k] || this._fkey_ids[k] !== new_fkeys[k].id){
                    changing.add(k);
                    if(new_fkeys[k]){
                        this._files[k] = new_fkeys[k].file || new_fkeys[k]; // cut sometimes isn't an fkey but a CutStackState
                        this._fkey_ids[k] = new_fkeys[k].id;       
                    } else {
                        this._files[k] = null;
                        this._fkey_ids[k] = null;      
                    }
                }
            }
            if(changing.has('tet') && this._files.cut && this._files.cut.is === "n"){
                changing.add('cut');
            }
            
            if(changing.has('set')){
                this._setDuration(null);
                this._setSet_header(null);
            }
            if(changing.has('pos')){
                this._setPos_xy(null);
                this._setPos_speed(null);
                this._setPos_dir(null);
                this._setPos_header(null);
            }
            if(changing.has('tet')){
                this._setSpike_times(null);
                this._setAmplitudes(null);
                this._setData_for_gl(null);
                this._setTet_header(null);
            }
            if(changing.has('cut')){
                this._setCut_box(null);
            }

            // build request for io-worker, taking as much as possible from the main cache
            var request = {};
            var options = {};

            if(changing.has('set')){
                request['set'] = null;
                if(this._files.set && this._set_from_fkey.has(this._fkey_ids.set)){
                    this._use_set(this._set_from_fkey.get(this._fkey_ids.set));
                } else {
                    request['set'] = this._files.set;
                    request['set_fkey'] = this._fkey_ids.set;
                }
            }

            if(changing.has('cut')){
                this._make_cut_from_n = false;
                request['cut'] = null;
                if(!this._files.cut){
                    this._make_cut_from_n = true; // we have already guaranteed that changing has 'tet'
                                                  // though if there's no tet file we will never get to make the cut
                } else if(this._files.cut.is === "stack"){
                    // new_values.cut is an fkey for a "fake" file, which is actually a cut "stack" object
                    this._setCut_box(this._files.cut); //easy
                } else {
                    // this._files.cut is an fkey for a proper file
                    if(this._cut_from_fkey.has(this._fkey_ids.cut)){
                        // create box from cached inds and header
                        this._setCut_box({
                            is: 'file',
                            akey: this._cut_from_fkey.get(this._fkey_ids.cut).cut,
                            header: this._cut_from_fkey.get(this._fkey_ids.cut).header
                        });
                    } else {
                        request['cut'] = this._files.cut;
                        request['cut_fkey'] = this._fkey_ids.cut;
                    }
                }
            }

            if(changing.has('pos')){
                request['pos'] = null;
                if(this._files.pos){
                    var in_cache = false;
                    var pos_options = this._make_pos_options();
                    pos_options.fkey = this._fkey_ids.pos;
                    for(let [cached_opts, data] of this._pos_from_options){
                        if(Utils.is_equal_simple(cached_opts, pos_options)){
                            this._use_pos(data);
                            in_cache = true;
                            break;
                        }
                    }
                    if(!in_cache){
                        request['pos'] = this._files.pos;
                        options['pos'] = this._make_pos_options();
                        request['pos_fkey'] = this._fkey_ids.pos;
                    }
                }
            }

            if(changing.has('tet')){
                this._make_tet_request(this._fkey_ids.tet, this._files.tet, request, options);
            }

            this._worker_io.exec('read_files', request, options); 

            this._setNo_groups(!(new_fkeys.tet || new_fkeys.cut)); // as promissed, we only set this once during the file-change

        }, _pos_options_changed: function(){
            var options = this._make_pos_options();
            options.fkey = this._fkey_ids.pos;
            for(let [cached_opts, data] of this._pos_from_options){
                if(Utils.is_equal_simple(cached_opts, options)){
                    this._use_pos(data);
                    return;
                }
            }
            delete options.fkey;
            this._worker_io.exec('read_files', {
                pos: this._files.pos,
                pos_fkey: this._fkey_ids.pos
            }, {
                pos: options
            });
        }, got_pos: function(msg, arrays){
            var am = Utils.typed_array_manager;
            var result = {
                xy: am.to_akey(arrays['xy']),
                dir: am.to_akey(arrays['dir']),
                speed: am.to_akey(arrays['speed']),
                header: msg.header
            };
            msg.options.fkey = msg.fkey; // needed both for adding to cache (on next line) and for delete_from_cache loop
            this._pos_from_options.set(msg.options, result);
            if(msg.delete_from_cache){
                for(let cached_opts of this._pos_from_options.keys()){
                    if(Utils.is_equal_simple(cached_opts, msg.delete_from_cache)){
                        this._pos_from_options.delete(cached_opts);
                    }
                }
            }
            var opts =  this._make_pos_options();
            opts.fkey = this._fkey_ids.pos;
            if(Utils.is_equal_simple(msg.options, opts)){
                this._use_pos(result);
            }            
        },
        _use_pos: function(result){
            let timebase = result.header && parseInt(result.header.timebase);
            this._setPos_xy({
                xy: result.xy,
                timebase: timebase,
                units_per_cm: result.header && parseInt(result.header.units_per_meter)/100,
                max_x: result.header && result.header.max_vals[0],
                max_y: result.header && result.header.max_vals[1]
            });
            this._setPos_dir({
                dir: result.dir,
                timebase: timebase
            });
            this._setPos_speed({
                speed: result.speed,
                timebase: timebase
            });
            this._setPos_header(result.header);
        },
        _make_tet_request: function(fkey, file, request, options){
            if(!fkey){
                request['tet'] = null;
                return;
            }

            var cached = this._tet_from_fkey.get(fkey) || {};
            options['tet'] = {};
            if(this.want_amplitudes.length){
                if(cached.amps){
                    this._setAmplitudes(cached.amps);
                } else {
                    request['tet'] = file;
                    request['tet_fkey'] = fkey;
                    options['tet']['amps'] = 1;    
                }
            } 
            if(this.want_spike_times.length){
                if(cached.times){
                    this._setSpike_times(cached.times);
                } else {
                    request['tet'] = file;
                    request['tet_fkey'] = fkey;
                    options['tet']['times'] = 1;
                }
            }
            if(this.want_data_for_gl.length){
                if(cached.gl_data) {
                    this._setData_for_gl(cached.gl_data);
                } else {
                    request['tet'] = file;
                    request['tet_fkey'] = fkey;
                    options['tet']['gl_data'] = 1;                            
                }
            }
            if(cached.header){
                this._setTet_header(cached.header);
                if(this._make_cut_from_n){
                    this._setCut_box({
                        is: 'n',
                        n_spikes: parseInt(cached.header.num_spikes),
                        header: null
                    });
                }

            }

        }, _tet_options_changed: function(){
            var request = {}, options = {};
            this._make_tet_request(this._fkey_ids.tet, this._files.tet, request, options);
            this._worker_io.exec('read_files', request, options); 
        }, cut_file_parsed: function(msg, arrays){
            var am = Utils.typed_array_manager;
            this._cut_from_fkey.set(msg.fkey,{
                header: msg.header,
                cut: am.to_akey(arrays['cut'])
            });
            if(this._fkey_ids.cut === msg.fkey){
                this._setCut_box({
                    is: 'file',
                    akey: this._cut_from_fkey.get(msg.fkey).cut,
                    header: this._cut_from_fkey.get(msg.fkey).header
                });
            }
            if(msg.delete_from_cache){
                this._cut_from_fkey.delete(msg.delete_from_cache);
            }
        }, _make_pos_options: function(){
            return {
                use_both_leds: (this.use_n_leds == 2),
                need_to_subtract_mins: false,
                swapping_thresh_cm: 1,
                max_speed: this.speed_filter_mps,
                smoothing_secs: this.pos_smoothing_s
            }
        }, got_gl_data: function(msg, arrays){
            var am = Utils.typed_array_manager;
            var cached = this._tet_from_fkey.get(msg.fkey);
            if(!cached){
                cached = {};
                this._tet_from_fkey.set(msg.fkey, cached);
            }
            cached.gl_data = {
                n: msg.n_spikes,
                voltages: am.to_akey(arrays.data)
            };
            if(msg.fkey === this._fkey_ids.tet){
                this._setData_for_gl(cached.gl_data);
            }
        }, got_amplitudes: function(msg, arrays){
            var am = Utils.typed_array_manager;
            var cached = this._tet_from_fkey.get(msg.fkey);
            if(!cached){
                cached = {};
                this._tet_from_fkey.set(msg.fkey, cached);
            }
            cached.amps = {
                n: msg.n_spikes,
                amps: am.to_akey(arrays.data),
                chan_is_ok: msg.chan_is_ok
            };
            if(msg.fkey === this._fkey_ids.tet){
                this._setAmplitudes(cached.amps);
            }
        }, got_times: function(msg, arrays){
            var am = Utils.typed_array_manager;
            var cached = this._tet_from_fkey.get(msg.fkey);
            if(!cached){
                cached = {};
                this._tet_from_fkey.set(msg.fkey, cached);
            }
            cached.times = {
                n: msg.n_spikes,
                timebase: msg.timebase,
                times: am.to_akey(arrays.data)
            };
            if(msg.fkey === this._fkey_ids.tet){
                this._setSpike_times(cached.times);
            }
        }, got_tet_header: function(fkey, header){
            var cached = this._tet_from_fkey.get(fkey);
            if(!cached){
                cached = {};
                this._tet_from_fkey.set(fkey, cached);
            }
            cached.header = header;
            if(fkey === this._fkey_ids.tet){
                this._setTet_header(header);
                if(this._make_cut_from_n){
                    this._setCut_box({
                        is: 'n',
                        n_spikes: parseInt(header.num_spikes),
                        header: null
                    })
                }
            }
        }, delete_from_tet_cache: function(fkey){
            this._tet_from_fkey.delete(fkey);
        }, set_file_parsed: function(msg) {
            if(this._fkey_ids.set === msg.fkey){
                this._use_set(msg.header);
            }
            this._set_from_fkey.set(msg.fkey, msg.header);
            if(msg.delete_from_cache){
                this._set_from_fkey.delete(msg.delete_from_cache);
            }
        }, _use_set: function(header){
            this._setDuration(parseInt(header.duration));
            let start = parseInt(header.lasttrialdatetime)*1000;
            this._setStart_end_times({
                start: start,
                end: start + this.duration *1000
            });
            this._setSet_header(header);
        }
    });
</script>

  
</dom-module>