github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/client/android/android.go (about) 1 /* 2 Copyright 2013 The Camlistore Authors 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package android contains code specific to running the Camlistore client 18 // code as a child process on Android. This removes ugly API from the 19 // client package itself. 20 package android 21 22 import ( 23 "bufio" 24 "crypto/tls" 25 "crypto/x509" 26 "fmt" 27 "io" 28 "io/ioutil" 29 "log" 30 "net" 31 "net/http" 32 "os" 33 "os/exec" 34 "path/filepath" 35 "regexp" 36 "runtime" 37 "runtime/debug" 38 "strconv" 39 "sync" 40 "time" 41 42 "camlistore.org/pkg/blob" 43 "camlistore.org/pkg/blobserver" 44 "camlistore.org/pkg/osutil" 45 "camlistore.org/pkg/schema" 46 ) 47 48 // TODO(mpl): distinguish CAMPUT, CAMGET, etc 49 var androidOutput, _ = strconv.ParseBool(os.Getenv("CAMPUT_ANDROID_OUTPUT")) 50 51 // IsChild reports whether this process is running as an Android 52 // child process and should report its output in the form that the 53 // Android uploader app expects. 54 func IsChild() bool { 55 memOnce.Do(startMemGoroutine) 56 return androidOutput 57 } 58 59 func PreExit() { 60 if !IsChild() { 61 return 62 } 63 Printf("STAT exit 1\n") 64 } 65 66 var androidOutMu sync.Mutex 67 68 func Printf(format string, args ...interface{}) { 69 androidOutMu.Lock() 70 defer androidOutMu.Unlock() 71 fmt.Printf(format, args...) 72 } 73 74 var detectOnce sync.Once 75 var onAndroidCache bool 76 77 func dirExists(f string) bool { 78 fi, err := os.Stat(f) 79 return err == nil && fi.IsDir() 80 } 81 82 func initOnAndroid() { 83 // Good enough heuristic. Suggestions welcome. 84 onAndroidCache = dirExists("/data/data") && dirExists("/system/etc") 85 } 86 87 func OnAndroid() bool { 88 detectOnce.Do(initOnAndroid) 89 return onAndroidCache 90 } 91 92 var pingRx = regexp.MustCompile(`\((.+?)\)`) 93 94 type namedInt struct { 95 name string 96 sync.Mutex 97 val int64 98 } 99 100 func (ni *namedInt) Incr(delta int64) { 101 ni.Lock() 102 ni.val += delta 103 nv := ni.val 104 ni.Unlock() 105 Printf("STAT %s %d\n", ni.name, nv) 106 } 107 108 func (ni *namedInt) Set(v int64) { 109 ni.Lock() 110 if v == ni.val { 111 ni.Unlock() 112 return 113 } 114 ni.val = v 115 ni.Unlock() 116 Printf("STAT %s %d\n", ni.name, v) 117 } 118 119 var ( 120 statDNSStart = &namedInt{name: "dns_start"} 121 statDNSDone = &namedInt{name: "dns_done"} 122 statTCPStart = &namedInt{name: "tcp_start"} 123 statTCPStarted = &namedInt{name: "tcp_started"} 124 statTCPFail = &namedInt{name: "tcp_fail"} 125 statTCPDone = &namedInt{name: "tcp_done_because_close"} 126 statTCPDoneRead = &namedInt{name: "tcp_done_because_read"} 127 statTCPWrites = &namedInt{name: "tcp_write_byte"} 128 statTCPWrote = &namedInt{name: "tcp_wrote_byte"} 129 statTCPReads = &namedInt{name: "tcp_read_byte"} 130 statHTTPStart = &namedInt{name: "http_start"} 131 statHTTPResHeaders = &namedInt{name: "http_res_headers"} 132 statBlobUploaded = &namedInt{name: "blob_uploaded"} 133 statBlobExisted = &namedInt{name: "blob_existed"} 134 statFileUploaded = &namedInt{name: "file_uploaded"} 135 statFileExisted = &namedInt{name: "file_existed"} 136 statMemReleased = &namedInt{name: "mem_heap_released"} 137 statMemAlloc = &namedInt{name: "mem_alloc"} 138 statMemRSS = &namedInt{name: "mem_rss"} 139 ) 140 141 type statTrackingConn struct { 142 net.Conn 143 once sync.Once // guards close stats 144 } 145 146 func (c *statTrackingConn) Write(p []byte) (n int, err error) { 147 statTCPWrites.Incr(int64(len(p))) 148 n, err = c.Conn.Write(p) 149 statTCPWrote.Incr(int64(n)) 150 return 151 } 152 153 func (c *statTrackingConn) Read(p []byte) (n int, err error) { 154 n, err = c.Conn.Read(p) 155 statTCPReads.Incr(int64(n)) 156 if err != nil { 157 c.once.Do(func() { 158 statTCPDoneRead.Incr(1) 159 }) 160 } 161 return 162 } 163 164 func (c *statTrackingConn) Close() error { 165 c.once.Do(func() { 166 statTCPDone.Incr(1) 167 }) 168 return nil 169 } 170 171 var ( 172 dnsMu sync.Mutex 173 dnsCache = make(map[string]string) 174 ) 175 176 func androidLookupHost(host string) string { 177 dnsMu.Lock() 178 v, ok := dnsCache[host] 179 dnsMu.Unlock() 180 if ok { 181 return v 182 } 183 statDNSStart.Incr(1) 184 defer statDNSDone.Incr(1) 185 186 // Android has no "dig" or "host" tool, so use "ping -c 1". Ghetto. 187 // $ ping -c 1 google.com 188 // PING google.com (74.125.224.64) 56(84) bytes of data. 189 c := make(chan string, 1) 190 go func() { 191 cmd := exec.Command("/system/bin/ping", "-c", "1", host) 192 stdout, err := cmd.StdoutPipe() 193 if err != nil { 194 panic(err) 195 } 196 defer stdout.Close() 197 if err := cmd.Start(); err != nil { 198 log.Printf("Error resolving %q with ping: %v", host, err) 199 c <- host 200 return 201 } 202 defer func() { 203 if p := cmd.Process; p != nil { 204 p.Kill() 205 } 206 }() 207 br := bufio.NewReader(stdout) 208 line, err := br.ReadString('\n') 209 if err != nil { 210 log.Printf("Failed to resolve %q with ping", host) 211 c <- host 212 return 213 } 214 if m := pingRx.FindStringSubmatch(line); m != nil { 215 ip := m[1] 216 dnsMu.Lock() 217 dnsCache[host] = ip 218 dnsMu.Unlock() 219 c <- ip 220 return 221 } 222 log.Printf("Failed to resolve %q with ping", host) 223 c <- host 224 }() 225 return <-c 226 return v 227 } 228 229 type StatsTransport struct { 230 Rt http.RoundTripper 231 } 232 233 func (t StatsTransport) RoundTrip(req *http.Request) (res *http.Response, err error) { 234 statHTTPStart.Incr(1) 235 res, err = t.Rt.RoundTrip(req) 236 statHTTPResHeaders.Incr(1) 237 // TODO: track per-response code stats, and also track when body done being read. 238 return 239 } 240 241 func Dial(network, addr string) (net.Conn, error) { 242 // Temporary laziness hack, avoiding doing a 243 // cross-compiled Android cgo build. 244 // Without cgo, package net uses 245 // /etc/resolv.conf (not available on 246 // Android). We really want a cgo binary to 247 // use Android's DNS cache, but it's kinda 248 // hard/impossible to cross-compile for now. 249 statTCPStart.Incr(1) 250 host, port, err := net.SplitHostPort(addr) 251 if err != nil { 252 statTCPFail.Incr(1) 253 return nil, fmt.Errorf("couldn't split %q", addr) 254 } 255 if OnAndroid() { 256 // Only do the Android DNS lookups when actually 257 // running on a device. We also run in "Android mode" 258 // (IsChild) in tests and interactive debugging. 259 host = androidLookupHost(host) 260 } 261 c, err := net.Dial(network, net.JoinHostPort(host, port)) 262 if err != nil { 263 statTCPFail.Incr(1) 264 return nil, err 265 } 266 statTCPStarted.Incr(1) 267 return &statTrackingConn{Conn: c}, err 268 } 269 270 func TLSConfig() (*tls.Config, error) { 271 if !OnAndroid() { 272 return nil, nil 273 } 274 certDir := "/system/etc/security/cacerts" 275 fi, err := os.Stat(certDir) 276 if err != nil { 277 return nil, err 278 } 279 if !fi.IsDir() { 280 return nil, fmt.Errorf("%q not a dir", certDir) 281 } 282 pool := x509.NewCertPool() 283 cfg := &tls.Config{RootCAs: pool} 284 285 f, err := os.Open(certDir) 286 if err != nil { 287 return nil, err 288 } 289 defer f.Close() 290 names, _ := f.Readdirnames(-1) 291 for _, name := range names { 292 pem, err := ioutil.ReadFile(filepath.Join(certDir, name)) 293 if err != nil { 294 return nil, err 295 } 296 pool.AppendCertsFromPEM(pem) 297 } 298 return cfg, nil 299 } 300 301 // NoteFileUploaded is a hook for camput to report that a file 302 // was uploaded. TODO: move this to pkg/client/android probably. 303 func NoteFileUploaded(fullPath string, uploaded bool) { 304 if !IsChild() { 305 return 306 } 307 if uploaded { 308 statFileUploaded.Incr(1) 309 } else { 310 statFileExisted.Incr(1) 311 } 312 Printf("FILE_UPLOADED %s\n", fullPath) 313 } 314 315 // androidStatusReceiver is a blobserver.StatReceiver wrapper that 316 // reports the full filename path and size of uploaded blobs. 317 // The android app wrapping camput watches stdout for this, for progress bars. 318 type StatusReceiver struct { 319 Sr blobserver.StatReceiver 320 Path string 321 } 322 323 func (asr StatusReceiver) noteChunkOnServer(sb blob.SizedRef) { 324 Printf("CHUNK_UPLOADED %d %s %s\n", sb.Size, sb.Ref, asr.Path) 325 } 326 327 func (asr StatusReceiver) ReceiveBlob(blob blob.Ref, source io.Reader) (blob.SizedRef, error) { 328 // Sniff the first 1KB of it and don't print the stats if it looks like it was just a schema 329 // blob. We won't update the progress bar for that yet. 330 var buf [1024]byte 331 contents := buf[:0] 332 sb, err := asr.Sr.ReceiveBlob(blob, io.TeeReader(source, writeUntilSliceFull{&contents})) 333 if err == nil && !schema.LikelySchemaBlob(contents) { 334 statBlobUploaded.Incr(1) 335 asr.noteChunkOnServer(sb) 336 } 337 return sb, err 338 } 339 340 func (asr StatusReceiver) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error { 341 midc := make(chan blob.SizedRef) 342 errc := make(chan error, 1) 343 go func() { 344 err := asr.Sr.StatBlobs(midc, blobs) 345 errc <- err 346 close(midc) 347 }() 348 for sb := range midc { 349 asr.noteChunkOnServer(sb) 350 statBlobExisted.Incr(1) 351 dest <- sb 352 } 353 return <-errc 354 } 355 356 type writeUntilSliceFull struct { 357 s *[]byte 358 } 359 360 func (w writeUntilSliceFull) Write(p []byte) (n int, err error) { 361 s := *w.s 362 l := len(s) 363 growBy := cap(s) - l 364 if growBy > len(p) { 365 growBy = len(p) 366 } 367 s = s[0 : l+growBy] 368 copy(s[l:], p) 369 *w.s = s 370 return len(p), nil 371 } 372 373 var memOnce sync.Once 374 375 func startMemGoroutine() { 376 if !androidOutput { 377 return 378 } 379 go func() { 380 var ms runtime.MemStats 381 n := 0 382 for { 383 runtime.ReadMemStats(&ms) 384 statMemReleased.Set(int64(ms.HeapReleased)) 385 statMemAlloc.Set(int64(ms.Alloc)) 386 statMemRSS.Set(osutil.MemUsage()) 387 time.Sleep(1 * time.Second) 388 n++ 389 if n%5 == 0 { 390 debug.FreeOSMemory() 391 } 392 } 393 }() 394 }