github.com/slspeek/camlistore_namedsearch@v0.0.0-20140519202248-ed6f70f7721a/pkg/client/android/android.go (about)

     1  /*
     2  Copyright 2013 The Camlistore Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package android contains code specific to running the Camlistore client
    18  // code as a child process on Android. This removes ugly API from the
    19  // client package itself.
    20  package android
    21  
    22  import (
    23  	"bufio"
    24  	"crypto/tls"
    25  	"crypto/x509"
    26  	"fmt"
    27  	"io"
    28  	"io/ioutil"
    29  	"log"
    30  	"net"
    31  	"net/http"
    32  	"os"
    33  	"os/exec"
    34  	"path/filepath"
    35  	"regexp"
    36  	"runtime"
    37  	"runtime/debug"
    38  	"strconv"
    39  	"sync"
    40  	"time"
    41  
    42  	"camlistore.org/pkg/blob"
    43  	"camlistore.org/pkg/blobserver"
    44  	"camlistore.org/pkg/osutil"
    45  	"camlistore.org/pkg/schema"
    46  )
    47  
    48  // TODO(mpl): distinguish CAMPUT, CAMGET, etc
    49  var androidOutput, _ = strconv.ParseBool(os.Getenv("CAMPUT_ANDROID_OUTPUT"))
    50  
    51  // IsChild reports whether this process is running as an Android
    52  // child process and should report its output in the form that the
    53  // Android uploader app expects.
    54  func IsChild() bool {
    55  	memOnce.Do(startMemGoroutine)
    56  	return androidOutput
    57  }
    58  
    59  func PreExit() {
    60  	if !IsChild() {
    61  		return
    62  	}
    63  	Printf("STAT exit 1\n")
    64  }
    65  
    66  var androidOutMu sync.Mutex
    67  
    68  func Printf(format string, args ...interface{}) {
    69  	androidOutMu.Lock()
    70  	defer androidOutMu.Unlock()
    71  	fmt.Printf(format, args...)
    72  }
    73  
    74  var detectOnce sync.Once
    75  var onAndroidCache bool
    76  
    77  func dirExists(f string) bool {
    78  	fi, err := os.Stat(f)
    79  	return err == nil && fi.IsDir()
    80  }
    81  
    82  func initOnAndroid() {
    83  	// Good enough heuristic. Suggestions welcome.
    84  	onAndroidCache = dirExists("/data/data") && dirExists("/system/etc")
    85  }
    86  
    87  func OnAndroid() bool {
    88  	detectOnce.Do(initOnAndroid)
    89  	return onAndroidCache
    90  }
    91  
    92  var pingRx = regexp.MustCompile(`\((.+?)\)`)
    93  
    94  type namedInt struct {
    95  	name string
    96  	sync.Mutex
    97  	val int64
    98  }
    99  
   100  func (ni *namedInt) Incr(delta int64) {
   101  	ni.Lock()
   102  	ni.val += delta
   103  	nv := ni.val
   104  	ni.Unlock()
   105  	Printf("STAT %s %d\n", ni.name, nv)
   106  }
   107  
   108  func (ni *namedInt) Set(v int64) {
   109  	ni.Lock()
   110  	if v == ni.val {
   111  		ni.Unlock()
   112  		return
   113  	}
   114  	ni.val = v
   115  	ni.Unlock()
   116  	Printf("STAT %s %d\n", ni.name, v)
   117  }
   118  
   119  var (
   120  	statDNSStart       = &namedInt{name: "dns_start"}
   121  	statDNSDone        = &namedInt{name: "dns_done"}
   122  	statTCPStart       = &namedInt{name: "tcp_start"}
   123  	statTCPStarted     = &namedInt{name: "tcp_started"}
   124  	statTCPFail        = &namedInt{name: "tcp_fail"}
   125  	statTCPDone        = &namedInt{name: "tcp_done_because_close"}
   126  	statTCPDoneRead    = &namedInt{name: "tcp_done_because_read"}
   127  	statTCPWrites      = &namedInt{name: "tcp_write_byte"}
   128  	statTCPWrote       = &namedInt{name: "tcp_wrote_byte"}
   129  	statTCPReads       = &namedInt{name: "tcp_read_byte"}
   130  	statHTTPStart      = &namedInt{name: "http_start"}
   131  	statHTTPResHeaders = &namedInt{name: "http_res_headers"}
   132  	statBlobUploaded   = &namedInt{name: "blob_uploaded"}
   133  	statBlobExisted    = &namedInt{name: "blob_existed"}
   134  	statFileUploaded   = &namedInt{name: "file_uploaded"}
   135  	statFileExisted    = &namedInt{name: "file_existed"}
   136  	statMemReleased    = &namedInt{name: "mem_heap_released"}
   137  	statMemAlloc       = &namedInt{name: "mem_alloc"}
   138  	statMemRSS         = &namedInt{name: "mem_rss"}
   139  )
   140  
   141  type statTrackingConn struct {
   142  	net.Conn
   143  	once sync.Once // guards close stats
   144  }
   145  
   146  func (c *statTrackingConn) Write(p []byte) (n int, err error) {
   147  	statTCPWrites.Incr(int64(len(p)))
   148  	n, err = c.Conn.Write(p)
   149  	statTCPWrote.Incr(int64(n))
   150  	return
   151  }
   152  
   153  func (c *statTrackingConn) Read(p []byte) (n int, err error) {
   154  	n, err = c.Conn.Read(p)
   155  	statTCPReads.Incr(int64(n))
   156  	if err != nil {
   157  		c.once.Do(func() {
   158  			statTCPDoneRead.Incr(1)
   159  		})
   160  	}
   161  	return
   162  }
   163  
   164  func (c *statTrackingConn) Close() error {
   165  	c.once.Do(func() {
   166  		statTCPDone.Incr(1)
   167  	})
   168  	return nil
   169  }
   170  
   171  var (
   172  	dnsMu    sync.Mutex
   173  	dnsCache = make(map[string]string)
   174  )
   175  
   176  func androidLookupHost(host string) string {
   177  	dnsMu.Lock()
   178  	v, ok := dnsCache[host]
   179  	dnsMu.Unlock()
   180  	if ok {
   181  		return v
   182  	}
   183  	statDNSStart.Incr(1)
   184  	defer statDNSDone.Incr(1)
   185  
   186  	// Android has no "dig" or "host" tool, so use "ping -c 1". Ghetto.
   187  	// $ ping -c 1 google.com
   188  	// PING google.com (74.125.224.64) 56(84) bytes of data.
   189  	c := make(chan string, 1)
   190  	go func() {
   191  		cmd := exec.Command("/system/bin/ping", "-c", "1", host)
   192  		stdout, err := cmd.StdoutPipe()
   193  		if err != nil {
   194  			panic(err)
   195  		}
   196  		defer stdout.Close()
   197  		if err := cmd.Start(); err != nil {
   198  			log.Printf("Error resolving %q with ping: %v", host, err)
   199  			c <- host
   200  			return
   201  		}
   202  		defer func() {
   203  			if p := cmd.Process; p != nil {
   204  				p.Kill()
   205  			}
   206  		}()
   207  		br := bufio.NewReader(stdout)
   208  		line, err := br.ReadString('\n')
   209  		if err != nil {
   210  			log.Printf("Failed to resolve %q with ping", host)
   211  			c <- host
   212  			return
   213  		}
   214  		if m := pingRx.FindStringSubmatch(line); m != nil {
   215  			ip := m[1]
   216  			dnsMu.Lock()
   217  			dnsCache[host] = ip
   218  			dnsMu.Unlock()
   219  			c <- ip
   220  			return
   221  		}
   222  		log.Printf("Failed to resolve %q with ping", host)
   223  		c <- host
   224  	}()
   225  	return <-c
   226  	return v
   227  }
   228  
   229  type StatsTransport struct {
   230  	Rt http.RoundTripper
   231  }
   232  
   233  func (t StatsTransport) RoundTrip(req *http.Request) (res *http.Response, err error) {
   234  	statHTTPStart.Incr(1)
   235  	res, err = t.Rt.RoundTrip(req)
   236  	statHTTPResHeaders.Incr(1)
   237  	// TODO: track per-response code stats, and also track when body done being read.
   238  	return
   239  }
   240  
   241  func Dial(network, addr string) (net.Conn, error) {
   242  	// Temporary laziness hack, avoiding doing a
   243  	// cross-compiled Android cgo build.
   244  	// Without cgo, package net uses
   245  	// /etc/resolv.conf (not available on
   246  	// Android).  We really want a cgo binary to
   247  	// use Android's DNS cache, but it's kinda
   248  	// hard/impossible to cross-compile for now.
   249  	statTCPStart.Incr(1)
   250  	host, port, err := net.SplitHostPort(addr)
   251  	if err != nil {
   252  		statTCPFail.Incr(1)
   253  		return nil, fmt.Errorf("couldn't split %q", addr)
   254  	}
   255  	if OnAndroid() {
   256  		// Only do the Android DNS lookups when actually
   257  		// running on a device. We also run in "Android mode"
   258  		// (IsChild) in tests and interactive debugging.
   259  		host = androidLookupHost(host)
   260  	}
   261  	c, err := net.Dial(network, net.JoinHostPort(host, port))
   262  	if err != nil {
   263  		statTCPFail.Incr(1)
   264  		return nil, err
   265  	}
   266  	statTCPStarted.Incr(1)
   267  	return &statTrackingConn{Conn: c}, err
   268  }
   269  
   270  func TLSConfig() (*tls.Config, error) {
   271  	if !OnAndroid() {
   272  		return nil, nil
   273  	}
   274  	certDir := "/system/etc/security/cacerts"
   275  	fi, err := os.Stat(certDir)
   276  	if err != nil {
   277  		return nil, err
   278  	}
   279  	if !fi.IsDir() {
   280  		return nil, fmt.Errorf("%q not a dir", certDir)
   281  	}
   282  	pool := x509.NewCertPool()
   283  	cfg := &tls.Config{RootCAs: pool}
   284  
   285  	f, err := os.Open(certDir)
   286  	if err != nil {
   287  		return nil, err
   288  	}
   289  	defer f.Close()
   290  	names, _ := f.Readdirnames(-1)
   291  	for _, name := range names {
   292  		pem, err := ioutil.ReadFile(filepath.Join(certDir, name))
   293  		if err != nil {
   294  			return nil, err
   295  		}
   296  		pool.AppendCertsFromPEM(pem)
   297  	}
   298  	return cfg, nil
   299  }
   300  
   301  // NoteFileUploaded is a hook for camput to report that a file
   302  // was uploaded.  TODO: move this to pkg/client/android probably.
   303  func NoteFileUploaded(fullPath string, uploaded bool) {
   304  	if !IsChild() {
   305  		return
   306  	}
   307  	if uploaded {
   308  		statFileUploaded.Incr(1)
   309  	} else {
   310  		statFileExisted.Incr(1)
   311  	}
   312  	Printf("FILE_UPLOADED %s\n", fullPath)
   313  }
   314  
   315  // androidStatusReceiver is a blobserver.StatReceiver wrapper that
   316  // reports the full filename path and size of uploaded blobs.
   317  // The android app wrapping camput watches stdout for this, for progress bars.
   318  type StatusReceiver struct {
   319  	Sr   blobserver.StatReceiver
   320  	Path string
   321  }
   322  
   323  func (asr StatusReceiver) noteChunkOnServer(sb blob.SizedRef) {
   324  	Printf("CHUNK_UPLOADED %d %s %s\n", sb.Size, sb.Ref, asr.Path)
   325  }
   326  
   327  func (asr StatusReceiver) ReceiveBlob(blob blob.Ref, source io.Reader) (blob.SizedRef, error) {
   328  	// Sniff the first 1KB of it and don't print the stats if it looks like it was just a schema
   329  	// blob.  We won't update the progress bar for that yet.
   330  	var buf [1024]byte
   331  	contents := buf[:0]
   332  	sb, err := asr.Sr.ReceiveBlob(blob, io.TeeReader(source, writeUntilSliceFull{&contents}))
   333  	if err == nil && !schema.LikelySchemaBlob(contents) {
   334  		statBlobUploaded.Incr(1)
   335  		asr.noteChunkOnServer(sb)
   336  	}
   337  	return sb, err
   338  }
   339  
   340  func (asr StatusReceiver) StatBlobs(dest chan<- blob.SizedRef, blobs []blob.Ref) error {
   341  	midc := make(chan blob.SizedRef)
   342  	errc := make(chan error, 1)
   343  	go func() {
   344  		err := asr.Sr.StatBlobs(midc, blobs)
   345  		errc <- err
   346  		close(midc)
   347  	}()
   348  	for sb := range midc {
   349  		asr.noteChunkOnServer(sb)
   350  		statBlobExisted.Incr(1)
   351  		dest <- sb
   352  	}
   353  	return <-errc
   354  }
   355  
   356  type writeUntilSliceFull struct {
   357  	s *[]byte
   358  }
   359  
   360  func (w writeUntilSliceFull) Write(p []byte) (n int, err error) {
   361  	s := *w.s
   362  	l := len(s)
   363  	growBy := cap(s) - l
   364  	if growBy > len(p) {
   365  		growBy = len(p)
   366  	}
   367  	s = s[0 : l+growBy]
   368  	copy(s[l:], p)
   369  	*w.s = s
   370  	return len(p), nil
   371  }
   372  
   373  var memOnce sync.Once
   374  
   375  func startMemGoroutine() {
   376  	if !androidOutput {
   377  		return
   378  	}
   379  	go func() {
   380  		var ms runtime.MemStats
   381  		n := 0
   382  		for {
   383  			runtime.ReadMemStats(&ms)
   384  			statMemReleased.Set(int64(ms.HeapReleased))
   385  			statMemAlloc.Set(int64(ms.Alloc))
   386  			statMemRSS.Set(osutil.MemUsage())
   387  			time.Sleep(1 * time.Second)
   388  			n++
   389  			if n%5 == 0 {
   390  				debug.FreeOSMemory()
   391  			}
   392  		}
   393  	}()
   394  }