Mic92/callgraph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

tune performance; more robust selnium

Browse Source
This commit is contained in:
Jörg Thalheim 2016-05-09 15:33:33 +00:00
parent b20a4a06d6
commit 6c88e1be1d
6 changed files with 215 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

89
benchmark.rb Normal file
View File

@ -0,0 +1,89 @@
#!/usr/bin/env ruby
require 'socket'
require 'timeout'
def start(*exe)
puts "Starting #{exe}"
pid = spawn(*exe, out: "/dev/null")
Process.detach(pid)
return pid
end
def is_port_open?(ip, port)
begin
Timeout::timeout(1) do
begin
s = TCPSocket.new(ip, port)
s.close
return true
rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
return false
end
end
rescue Timeout::Error
end
return false
end
def killall(pids)
pids.each do |pid|
puts "Killing #{pid}"
begin
Process.kill "TERM", pid
Process.wait pid
rescue => ex
puts "ERROR: Couldn't kill #{pid}. #{ex.class}=#{ex.message}"
end
end
end
def run
ab = ["ab", "-n", "10000", "http://0.0.0.0:9089/main.go"]
puts("$ " + ab.join(" "))
out = ""
IO.popen(ab) do |io|
out = io.read
puts(out)
out.split(/\n/).map do |l|
next unless l =~ /^Requests per second:\s+(\d+.\d+)/
return $1
end
end
abort("failed to parse ab output: #{out}")
end
def main()
pids = []
pids << start("docker", "run", "-p", "9089:80", "-v", "#{File.realpath(".")}/:/usr/share/nginx/html:ro", "--rm", "nginx:alpine")
open = false
10.times do
sleep(1)
open ||= is_port_open?("0.0.0.0", 9089)
rc = Process.waitpid(pids[0], Process::WNOHANG)
unless rc.nil?
abort("failed to start nginx container")
end
break if open
end
unless open
abort("failed to start nginx container")
end
run() # warmup
median1 = run()
pids << start("./callgraph", "-skip-test", "-rancher-host", "rancher.local:8080", "http://sharelatex.local/login", "127.0.0.1")
sleep(2)
median2 = run()
killall([pids.pop]) rescue nil
puts (pids)
pids << start("sudo", "tcpdump", "-i", "any", "-n")
sleep(2)
median3 = run()
puts "####median request time/s without, with sysdig and with tcpdump: #{median1} #{median2} #{median3}"
ensure
killall(pids)
end
main

68
capture-connections.lua
View File

@ -11,10 +11,10 @@ function on_init()
local mapping = {
isread = "evt.is_io_read",
buflen = "evt.buflen",
proc = "proc.name",
pid = "proc.pid",
tid = "thread.tid",
container = "container.name",
--proc = "proc.name",
--pid = "proc.pid",
--tid = "thread.tid",
--container = "container.name",
sip = "fd.sip",
sport = "fd.sport",
cip = "fd.cip",
@ -26,66 +26,46 @@ function on_init()
for k,v in pairs(mapping) do
fields[k] = chisel.request_field(v)
end
key_fields = {fields.container, fields.proc, fields.pid, fields.tid, fields.proto, fields.sip, fields.sport, fields.cip, fields.cport}
--key_fields = {fields.container, fields.proc, fields.pid, fields.tid, fields.proto, fields.sip, fields.sport, fields.cip, fields.cport}
key_fields = {fields.proto, fields.sip, fields.sport, fields.cip, fields.cport}
sysdig.set_snaplen(0)
chisel.set_filter("evt.is_io=true and (fd.type=ipv4 or fd.type=ipv6) and fd.rip exists and fd.lip exists and container.name!=host")
chisel.set_filter("evt.is_io=true and fd.rip exists and container.name!=host")
return true
end
local stats = {}
local DEBUG = false
-- localize function to save scope lookups
local table_concat = table.concat
local evt_field = evt.field
local ipairs = ipairs
function on_event()
-- only capture connections of servers
local sip = evt.field(fields.sip)
if not (evt.field(fields.lip) == sip or evt.field(fields.rip) == sip) then
local fields = fields
local sip = evt_field(fields.sip)
if not (evt_field(fields.lip) == sip or evt_field(fields.rip) == sip) then
return true
end
local dir
if evt.field(fields.isread) then
dir = "rx"
else
dir = "tx"
end
if DEBUG then
function to_s(v)
return (evt.field(v) or "nil").." "
end
io.write("DEBUG: ",
to_s(fields.container),
to_s(fields.proc),
to_s(fields.pid),
to_s(fields.tid),
to_s(fields.proto),
to_s(fields.sip),
to_s(fields.sport),
to_s(fields.cip),
to_s(fields.cport),
to_s(fields.lip),
dir, "\n")
end
local t = { }
for k,v in ipairs(key_fields) do
t[#t+1] = tostring(evt.field(v))
t[k] = tostring(evt_field(v))
end
t[#t+1] = dir
local key = table.concat(t, "\t")
if evt.field(fields.isread) then
t[#t+1] = "rx"
else
t[#t+1] = "tx"
end
local key = table_concat(t, "\t")
stats[key] = (stats[key] or 0) + (evt.field(fields.buflen) or 0)
stats[key] = (stats[key] or 0) + (evt_field(fields.buflen) or 0)
return true
end
function string.starts(string, prefix)
return string.sub(string, 1, string.len(start)) == prefix
end
function on_capture_start()
hostname = sysdig.get_machine_info().hostname
return true
@ -93,6 +73,6 @@ end
function on_capture_end()
for k, v in pairs(stats) do
io.write(hostname, "\t", k, "\t", v, "\n")
io.write(k, "\t", v, "\n")
end
end

22
main.go
View File

@ -6,6 +6,7 @@ import (
"net"
"os"
"os/exec"
"os/signal"
"syscall"
"time"
@ -15,6 +16,7 @@ import (
)
var (
skipTest = flag.Bool("skip-test", false, "skip selenium integration test")
rancherHost = flag.String("rancher-host", "localhost:8080", "host of rancher server")
rancherAccessKey = flag.String("rancher-access-key", os.Getenv("RANCHER_ACCESS_KEY"), "api access key")
rancherSecretKey = flag.String("rancher-secret-key", os.Getenv("RANCHER_SECRET_KEY"), "api secret key")
@ -22,7 +24,8 @@ var (
func parseArgs() (sharelatexUrl string, hosts []string) {
flag.Parse()
if len(os.Args) < 2 {
args := flag.Args()
if len(args) < 1 {
fmt.Fprintf(os.Stderr, "USAGE: %s SHARELATEXURL HOSTS...", os.Args[0])
os.Exit(1)
}
@ -34,8 +37,9 @@ func parseArgs() (sharelatexUrl string, hosts []string) {
fmt.Fprintf(os.Stderr, "Set RANCHER_SECRET_KEY environment variable")
os.Exit(1)
}
return os.Args[1], os.Args[2:]
return args[0], args[1:]
}
func rancherServices() ([]sysdig.Service, error) {
c := rancher.New(*rancherHost, *rancherAccessKey, *rancherSecretKey)
var api rancher.Api
@ -63,9 +67,12 @@ func rancherServices() ([]sysdig.Service, error) {
} else {
name = container.Labels.ProjectServiceName
}
if name == "Network Agent" {
continue
}
ip := net.ParseIP(container.PrimaryIpAddress)
if ip == nil {
return nil, fmt.Errorf("Got invalid IP from api for '%s' container: %s", container.Name, ip)
return nil, fmt.Errorf("Got invalid IP from api for '%s' container: %s", container.Name, container.PrimaryIpAddress)
}
serviceMap[name] = append(serviceMap[name], ip)
}
@ -89,12 +96,21 @@ func main() {
if err != nil {
die("failed to start sysdig: %v", err)
}
if *skipTest {
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, os.Interrupt)
signal.Notify(signalChan, syscall.SIGTERM)
<-signalChan
} else {
time.Sleep(4 * time.Second)
err = sharelatex.IntegrationTest(url)
if err != nil {
die("selenium not running?: %v", err)
}
time.Sleep(1 * time.Second)
}
cmd.SendStopSignal()
var records []sysdig.Record

82
sharelatex/client.go
View File

@ -11,6 +11,40 @@ import (
"github.com/tebeka/selenium"
)
type driver struct {
wd selenium.WebDriver
retries uint
}
func (d driver) FindXpath(selector string) selenium.WebElement {
return d.Find(selenium.ByXPATH, selector)
}
func (d driver) FindByCSS(selector string) selenium.WebElement {
return d.Find(selenium.ByCSSSelector, selector)
}
func (d driver) FindByName(selector string) selenium.WebElement {
return d.Find(selenium.ByName, selector)
}
func (d driver) FindByText(selector string) selenium.WebElement {
return d.Find(selenium.ByLinkText, selector)
}
func (d driver) Find(by, selector string) selenium.WebElement {
for i := uint(0); i < d.retries; i++ {
e, err := d.wd.FindElement(by, selector)
if err == nil {
return e
}
time.Sleep(time.Millisecond * 100)
}
fmt.Fprintf(os.Stderr, "elemenet %s not found (%s)", selector, by)
os.Exit(1)
return nil
}
func IntegrationTest(url string) error {
_, err := http.Get(url)
if err != nil {
@ -22,61 +56,61 @@ func IntegrationTest(url string) error {
return err
}
defer wd.Quit()
d := driver{wd, 100}
// Get simple playground interface
fmt.Fprintf(os.Stderr, "visit url %s\n", url)
wd.Get(url)
e, _ := wd.FindElement(selenium.ByXPATH, "//input[@type='email']")
e := d.FindXpath("//input[@type='email']")
e.Clear()
e.SendKeys("joerg@higgsboson.tk")
e, _ = wd.FindElement(selenium.ByXPATH, "//input[@type='password']")
e = d.FindXpath("//input[@type='password']")
e.Clear()
e.SendKeys("password")
e, _ = wd.FindElement(selenium.ByXPATH, "//button[@type='submit']")
e = d.FindXpath("//button[@type='submit']")
e.Click()
for {
_, err = wd.FindElement(selenium.ByLinkText, "Account")
if err == nil {
break
}
time.Sleep(time.Millisecond * 100)
}
d.FindByText("Account")
time.Sleep(time.Millisecond * 4000)
e, _ = wd.FindElement(selenium.ByLinkText, "New Project")
e = d.FindByText("New Project")
e.Click()
e, _ = wd.FindElement(selenium.ByLinkText, "Blank Project")
e = d.FindByText("Blank Project")
e.Click()
e, _ = wd.FindElement(selenium.ByXPATH, "//input[@type='text']")
e = d.FindXpath("//input[@type='text']")
e.Clear()
e.SendKeys(fmt.Sprintf("p %d", rand.Int63()))
e, _ = wd.FindElement(selenium.ByCSSSelector, "button.btn-primary") // Create
e.Click()
time.Sleep(time.Millisecond * 3000)
e, _ = wd.FindElement(selenium.ByCSSSelector, `i.fa-comment`)
e = d.FindByCSS("button.btn-primary") // Create
e.Click()
e, _ = wd.FindElement(selenium.ByCSSSelector, `div.new-message textarea`)
e = d.FindXpath("//a[@ng-controller='ShareController']") // Share
e.Click()
e = d.FindByCSS("button.btn-primary") // Close
e.Click()
e = d.FindByCSS(`i.fa-comment`)
e.Click()
e = d.FindByCSS(`div.new-message textarea`)
e.SendKeys("hello world" + selenium.EnterKey)
e, _ = wd.FindElement(selenium.ByCSSSelector, "i.fa.fa-upload") // Upload
e = d.FindByCSS("i.fa.fa-upload") // Upload
e.Click()
e, _ = wd.FindElement(selenium.ByName, "file") // Create
e = d.FindByName("file") // Create
dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
e.SendKeys(filepath.Join(dir, "./test.txt"))
time.Sleep(time.Millisecond * 500)
e, _ = wd.FindElement(selenium.ByCSSSelector, "textarea") // Latex Doc
e = d.FindByCSS("textarea") // Latex Doc
e.Clear()
e.SendKeys(`\documentclass{article} \begin{document} ello \end{document}`)
time.Sleep(time.Millisecond * 1000)
return nil
}

2
sysdig.sh
View File

@ -2,4 +2,4 @@
t="$(mktemp)"; f() {
rm "$t"
}
trap f EXIT; cat > "$t"; sudo sysdig -c "$t"
trap f EXIT; cat > "$t"; sudo sysdig -N -s0 -c "$t"

40
sysdig/records.go
View File

@ -16,8 +16,8 @@ const (
)
type Record struct {
Host, Container, Process string
Pid, Tid uint64
//Host, Container, Process string
//Pid, Tid uint64
Proto string
ServerIp net.IP
ServerPort uint16
@ -40,51 +40,35 @@ func ParseOutput(result string) ([]Record, error) {
if err != nil {
log.Fatal(err)
}
if record[0] == "host" {
continue
}
pid, err := strconv.ParseUint(record[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid pid %s: %v", record[2], err)
}
tid, err := strconv.ParseUint(record[4], 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid tid %s: %v", record[3], err)
}
sip := net.ParseIP(record[6])
sip := net.ParseIP(record[1])
if sip == nil {
return nil, fmt.Errorf("invalid ip '%s'", record[6])
return nil, fmt.Errorf("invalid ip '%s'", record[1])
}
sport, err := strconv.ParseUint(record[7], 10, 16)
sport, err := strconv.ParseUint(record[2], 10, 16)
if err != nil {
return nil, fmt.Errorf("invalid port %s: %v", record[7], err)
return nil, fmt.Errorf("invalid port %s: %v", record[2], err)
}
cip := net.ParseIP(record[8])
cip := net.ParseIP(record[3])
if cip == nil {
return nil, fmt.Errorf("invalid ip '%s'", record[8])
return nil, fmt.Errorf("invalid ip '%s'", record[3])
}
cport, err := strconv.ParseUint(record[9], 10, 16)
cport, err := strconv.ParseUint(record[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("invalid port %s: %v", record[9], err)
return nil, fmt.Errorf("invalid port %s: %v", record[4], err)
}
var direction int
if record[10] == "rx" {
if record[5] == "rx" {
direction = RX
} else {
direction = TX
}
bytes, err := strconv.ParseUint(record[11], 10, 64)
bytes, err := strconv.ParseUint(record[6], 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid transmission size %s: %v", record[11], err)
}
records = append(records, Record{
record[0],
record[1],
record[2],
pid,
tid,
record[5],
sip,
uint16(sport),
cip,