Skip to content

Commit

Permalink
Merge pull request torvalds#78 from sched-ext/scx-misc-updates
Browse files Browse the repository at this point in the history 
Misc updates to let scx_layered share ravg with rusty
  • Loading branch information
@htejun
htejun committed Nov 7, 2023
2 parents ee4efa7 + e322e56 commit 5a685d4
Show file tree
Hide file tree
Showing 5 changed files with 113 additions and 78 deletions.
5 changes: 3 additions & 2 deletions tools/sched_ext/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,9 @@ else
$(Q)cp "$(VMLINUX_H)" $@
endif

$(SCXOBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h user_exit_info.h \
| $(BPFOBJ) $(SCXOBJ_DIR)
$(SCXOBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h scx_common.bpf.h \
user_exit_info.h ravg.bpf.h ravg_impl.bpf.h \
| $(BPFOBJ) $(SCXOBJ_DIR)
$(call msg,CLNG-BPF,,$(notdir $@))
$(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@

Expand Down
82 changes: 82 additions & 0 deletions tools/sched_ext/ravg_read.rs.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/// ravg_read() implementation for rust userland. See ravg_read() in
/// ravg_impl.bpf.h. We don't yet have a good mechanism to share BPF and
/// matching rust code across multiple schedulers. For now, include both BPF
/// and rust code from scheduler implementations.
fn ravg_read(
val: u64,
val_at: u64,
old: u64,
cur: u64,
now: u64,
half_life: u32,
frac_bits: u32,
) -> f64 {
let ravg_1: f64 = (1 << frac_bits) as f64;
let half_life = half_life as u64;
let val = val as f64;
let mut old = old as f64 / ravg_1;
let mut cur = cur as f64 / ravg_1;

let now = now.max(val_at);
let normalized_dur = |dur| dur as f64 / half_life as f64;

//
// The following is f64 implementation of BPF ravg_accumulate().
//
let cur_seq = (now / half_life) as i64;
let val_seq = (val_at / half_life) as i64;
let seq_delta = (cur_seq - val_seq) as i32;

if seq_delta > 0 {
let full_decay = 2f64.powi(seq_delta);

// Decay $old and fold $cur into it.
old /= full_decay;
old += cur / full_decay;
cur = 0.0;

// Fold the oldest period whicy may be partial.
old += val * normalized_dur(half_life - val_at % half_life) / full_decay;

// Pre-computed decayed full-period values.
const FULL_SUMS: [f64; 20] = [
0.5,
0.75,
0.875,
0.9375,
0.96875,
0.984375,
0.9921875,
0.99609375,
0.998046875,
0.9990234375,
0.99951171875,
0.999755859375,
0.9998779296875,
0.99993896484375,
0.999969482421875,
0.9999847412109375,
0.9999923706054688,
0.9999961853027344,
0.9999980926513672,
0.9999990463256836,
// Use the same value beyond this point.
];

// Fold the full periods in the middle.
if seq_delta >= 2 {
let idx = ((seq_delta - 2) as usize).min(FULL_SUMS.len() - 1);
old += val * FULL_SUMS[idx];
}

// Accumulate the current period duration into @cur.
cur += val * normalized_dur(now % half_life);
} else {
cur += val * normalized_dur(now - val_at);
}

//
// The following is the blending part of BPF ravg_read().
//
old * (1.0 - normalized_dur(now % half_life) / 2.0) + cur / 2.0
}
3 changes: 2 additions & 1 deletion tools/sched_ext/scx_common.bpf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ BPF_PROG(name, ##args)
#define MEMBER_VPTR(base, member) (typeof((base) member) *)({ \
u64 __base = (u64)&(base); \
u64 __addr = (u64)&((base) member) - __base; \
_Static_assert(sizeof(base) >= sizeof((base) member)); \
_Static_assert(sizeof(base) >= sizeof((base) member), \
"@base is smaller than @member, is @base a pointer?"); \
asm volatile ( \
"if %0 <= %[max] goto +2\n" \
"%0 = 0\n" \
Expand Down
2 changes: 2 additions & 0 deletions tools/sched_ext/scx_rusty/src/bpf/rusty.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@
#define __kptr
#endif

#ifndef __KERNEL__
typedef unsigned char u8;
typedef unsigned int u32;
typedef unsigned long long u64;
#endif

#include "../../../ravg.bpf.h"

Expand Down
99 changes: 24 additions & 75 deletions tools/sched_ext/scx_rusty/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,12 @@ use log::trace;
use log::warn;
use ordered_float::OrderedFloat;

const RAVG_FRAC_BITS: u32 = rusty_sys::ravg_consts_RAVG_FRAC_BITS;
const MAX_DOMS: usize = rusty_sys::consts_MAX_DOMS as usize;
const MAX_CPUS: usize = rusty_sys::consts_MAX_CPUS as usize;

include!("../../ravg_read.rs.h");

/// scx_rusty is a multi-domain BPF / userspace hybrid scheduler where the BPF
/// part does simple round robin in each domain and the userspace part
/// calculates the load factor of each domain and tells the BPF part how to load
Expand Down Expand Up @@ -134,78 +137,6 @@ struct Opts {
verbose: u8,
}

fn ravg_read(rd: &rusty_sys::ravg_data, now: u64, half_life: u32) -> f64 {
const RAVG_1: f64 = (1 << rusty_sys::ravg_consts_RAVG_FRAC_BITS) as f64;
let half_life = half_life as u64;
let val = rd.val as f64;
let val_at = rd.val_at;
let mut old = rd.old as f64 / RAVG_1;
let mut cur = rd.cur as f64 / RAVG_1;

let now = now.max(val_at);
let normalized_dur = |dur| dur as f64 / half_life as f64;

//
// The following is f64 implementation of BPF ravg_accumulate().
//
let cur_seq = (now / half_life) as i64;
let val_seq = (val_at / half_life) as i64;
let seq_delta = (cur_seq - val_seq) as i32;

if seq_delta > 0 {
let full_decay = 2f64.powi(seq_delta);

// Decay $old and fold $cur into it.
old /= full_decay;
old += cur / full_decay;
cur = 0.0;

// Fold the oldest period whicy may be partial.
old += val * normalized_dur(half_life - val_at % half_life) / full_decay;

// Pre-computed decayed full-period values.
const FULL_SUMS: [f64; 20] = [
0.5,
0.75,
0.875,
0.9375,
0.96875,
0.984375,
0.9921875,
0.99609375,
0.998046875,
0.9990234375,
0.99951171875,
0.999755859375,
0.9998779296875,
0.99993896484375,
0.999969482421875,
0.9999847412109375,
0.9999923706054688,
0.9999961853027344,
0.9999980926513672,
0.9999990463256836,
// Use the same value beyond this point.
];

// Fold the full periods in the middle.
if seq_delta >= 2 {
let idx = ((seq_delta - 2) as usize).min(FULL_SUMS.len() - 1);
old += val * FULL_SUMS[idx];
}

// Accumulate the current period duration into @cur.
cur += val * normalized_dur(now % half_life);
} else {
cur += val * normalized_dur(now - val_at);
}

//
// The following is the blending part of BPF ravg_read().
//
old * (1.0 - normalized_dur(now % half_life) / 2.0) + cur / 2.0
}

fn now_monotonic() -> u64 {
let mut time = libc::timespec {
tv_sec: 0,
Expand Down Expand Up @@ -623,7 +554,16 @@ impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> {
&*(dom_ctx_map_elem.as_slice().as_ptr() as *const rusty_sys::dom_ctx)
};

self.dom_loads[i] = ravg_read(&dom_ctx.load_rd, now_mono, load_half_life);
let rd = &dom_ctx.load_rd;
self.dom_loads[i] = ravg_read(
rd.val,
rd.val_at,
rd.old,
rd.cur,
now_mono,
load_half_life,
RAVG_FRAC_BITS,
);

load_sum += self.dom_loads[i];
}
Expand Down Expand Up @@ -697,8 +637,17 @@ impl<'a, 'b, 'c> LoadBalancer<'a, 'b, 'c> {
continue;
}

let load =
task_ctx.weight as f64 * ravg_read(&task_ctx.dcyc_rd, now_mono, load_half_life);
let rd = &task_ctx.dcyc_rd;
let load = task_ctx.weight as f64
* ravg_read(
rd.val,
rd.val_at,
rd.old,
rd.cur,
now_mono,
load_half_life,
RAVG_FRAC_BITS,
);

tasks_by_load.insert(
OrderedFloat(load),
Expand Down

0 comments on commit 5a685d4

Please sign in to comment.