Merge pull request #5 from linuxserver/cosmetic_stuff

cosmetic changes using variables for all packages

.gitignore

@@ -47,3 +47,7 @@ app/config/settings.yml
 app/tmp
 tmp
 local-*.sh
+
+testconfig
+.idea
+

Dockerfile

@@ -1,166 +1,78 @@
-FROM lsiobase/xenial
+FROM lsiobase/ubuntu:xenial
-MAINTAINER phendryx
 
 # set version label
 ARG BUILD_DATE
 ARG VERSION
 LABEL build_version="Linuxserver.io version:- ${VERSION} Build-date:- ${BUILD_DATE}"
+LABEL maintainer="phendryx"
+
+# copy app files
+COPY app/ /opt/dockerhub-stats/
 
 # environment settings
 ARG DEBIAN_FRONTEND="noninteractive"
 
-# build packages as variable
+# package versions
+ARG GRAFANA_VER="4.1.2-1486989747"
+ARG INFLUX_VER="1.2.0"
+
+# packages as variables
 ARG BUILD_PACKAGES="\
-	apt-transport-https \
-	binutils \
-	bzip2 \
-	cpp \
-	cpp-5 \
-	dh-python \
-	distro-info-data \
 	g++ \
-	g++-5 \
 	gcc \
-	gcc-5 \
 	git \
-	git-core \
-	git-man \
-	icu-devtools \
-	libasan2 \
-	libatomic1 \
-	libc-dev-bin \
-	libc6-dev \
-	libcc1-0 \
-	libcilkrts5 \
-	liberror-perl \
-	libgcc-5-dev \
-	libgdbm3 \
-	libgmp-dev \
-	libgmpxx4ldbl \
-	libgomp1 \
-	libicu-dev \
-	libicu55 \
-	libisl15 \
-	libitm1 \
-	liblsan0 \
-	libmpc3 \
-	libmpdec2 \
-	libmpfr4 \
-	libmpx0 \
-	libperl5.22 \
-	libpng12-0 \
-	libpython3-stdlib \
-	libpython3.5-minimal \
-	libpython3.5-stdlib \
-	libquadmath0 \
-	libruby2.3 \
-	libstdc++-5-dev \
-	libtcl8.6 \
-	libtcltk-ruby \
-	libtk8.6 \
-	libtsan0 \
-	libubsan0 \
-	libx11-6 \
-	libx11-data \
-	libxau6 \
-	libxcb1 \
-	libxdmcp6 \
-	libxext6 \
-	libxml2 \
-	libxml2-dev \
-	libxrender1 \
-	libxslt1-dev \
-	libxslt1.1 \
-	libxss1 \
-	libyaml-0-2 \
-	linux-libc-dev \
-	lsb-release \
 	make \
-	mime-support \
-	perl \
-	perl-modules-5.22 \
-	python3 \
-	python3-minimal \
-	python3.5 \
-	python3.5-minimal \
-	rake \
-	ri \
-	ruby \
 	ruby-dev \
-	ruby-did-you-mean \
+	wget"
-	ruby-full \
+ARG RUNTIME_PACKAGES="\
-	ruby-minitest \
-	ruby-net-telnet \
-	ruby-power-assert \
-	ruby-test-unit \
-	ruby2.3 \
-	ruby2.3-dev \
-	ruby2.3-doc \
-	ruby2.3-tcltk \
-	rubygems-integration \
-	wget \
-	x11-common \
-	zlib1g-dev \
-	fontconfig-config \
-	fonts-dejavu-core \
-	libexpat1 \
-	libfontconfig1 \
-	libfontconfig \
-	libfreetype6 \
-	libpng12-0 \
-	ucf"
-
-# copy app and excludes
-COPY root/ /
-COPY app/ /opt/dockerhub-stats/
-
-# install build packages
-RUN \
- apt-get update && \
- apt-get install -y \
-	--no-install-recommends \
-	$BUILD_PACKAGES && \
-
-
-
-# install ruby app gems
-cd /opt/dockerhub-stats/ && \
- echo 'gem: --no-document' > \
-	/etc/gemrc && \
- gem install bundler && \
- bundle install && \
-
-# clean up
- apt-get purge -y --auto-remove \
-	$BUILD_PACKAGES && \
-
-# install runtime packages
- apt-get install -y \
-	--no-install-recommends \
 	cron \
 	libfontconfig1 \
 	libfreetype6 \
 	libpng12-0 \
 	netcat \
-	wget \
+	ruby \
-	ruby && \
+	wget"
-
+RUN \
- cd /tmp; wget wget https://dl.influxdata.com/influxdb/releases/influxdb_1.1.0_amd64.deb; dpkg -i influxdb_1.1.0_amd64.deb && \
+ echo "**** install build packages ****" && \
- cd /tmp; wget https://grafanarel.s3.amazonaws.com/builds/grafana_3.1.1-1470047149_amd64.deb; dpkg -i grafana_3.1.1-1470047149_amd64.deb && apt-get -f install && \
+ apt-get update && \
-
+ apt-get install -y \
+	--no-install-recommends \
+	$BUILD_PACKAGES && \
+ echo "**** install ruby app gems ****" && \
+ cd /opt/dockerhub-stats/ && \
+ echo 'gem: --no-document' > \
+	/etc/gemrc && \
+ gem install bundler && \
+ bundle install && \
+ echo "**** uninstall build packages ****" && \
+ apt-get purge -y --auto-remove \
+	$BUILD_PACKAGES && \
+ echo "****install runtime packages ****" && \
+ apt-get install -y \
+	--no-install-recommends \
+	$RUNTIME_PACKAGES && \
+ echo "**** install influxdb and grafana ****" && \
+ curl -o \
+ /tmp/influxdb.deb \
+	"https://dl.influxdata.com/influxdb/releases/influxdb_${INFLUX_VER}_amd64.deb" && \
+ dpkg -i /tmp/influxdb.deb && \
+ curl -o \
+ /tmp/grafana.deb \
+	"https://grafanarel.s3.amazonaws.com/builds/grafana_${GRAFANA_VER}_amd64.deb" && \
+ dpkg -i /tmp/grafana.deb && \
+ apt-get -f install && \
+ echo "**** cleanup ****" && \
  rm -rf \
+	/root \
 	/tmp/* \
 	/var/lib/apt/lists/* \
 	/var/tmp/* && \
- find /root -name . -o -prune -exec rm -rf -- {} + && \
- find /var/lib/gems -name "cache" -type d -exec rm -rf -- {} + && \
  mkdir -p \
 	/root
 
-# import grafana dashboard
+# copy local files
-# ADD ./grafana/dash.json /var/lib/grafana/dashboards/dash.json
+COPY root/ /
 
 # ports and volumes
-#WORKDIR /opt/docker-readme-sync
+EXPOSE 3000 8083 8086 4567
-EXPOSE 3000 8083 8086
 VOLUME /config

app/Gemfile

@@ -7,3 +7,4 @@ gem 'httparty', '~> 0.14.0'
 gem 'influxdb', '~> 0.3.12'
 
 gem 'awesome_print'
+gem 'sinatra', '~> 2.0', '>= 2.0.1'

app/Gemfile.lock

@@ -10,7 +10,17 @@ GEM
       json
     json (2.0.2)
     multi_xml (0.5.5)
+    mustermann (1.0.2)
+    rack (2.0.4)
+    rack-protection (2.0.1)
+      rack
     rake (11.3.0)
+    sinatra (2.0.1)
+      mustermann (~> 1.0)
+      rack (~> 2.0)
+      rack-protection (= 2.0.1)
+      tilt (~> 2.0)
+    tilt (2.0.8)
 
 PLATFORMS
   ruby
@@ -20,9 +30,10 @@ DEPENDENCIES
   httparty (~> 0.14.0)
   influxdb (~> 0.3.12)
   rake (~> 11.3)
+  sinatra (~> 2.0, >= 2.0.1)
 
 RUBY VERSION
    ruby 2.3.1p112
 
 BUNDLED WITH
-   1.13.6
+   1.16.1

app/stats.rb

@@ -1,40 +1,33 @@
-require "httparty"
 require "awesome_print"
 require "influxdb"
-require "yaml"
+require "sinatra"
 
-config_file = File.dirname(__FILE__) + "/config/settings.yml"
+def get_data
-config = YAML.load_file(config_file)
+  port = 8086
+  port = ENV['INFLUXDB_PORT'].to_i unless ENV['INFLUXDB_PORT'].nil?
 
-email = config["email"]
+  influxdb = InfluxDB::Client.new database: "dockerhub_stats", port: port
-password = config["password"]
+  query = 'select max(pull_count) as count from dockerhub_stats where time > now() - 1h group by repo,arch,org'
-org 	 = config["org"]
+  res = influxdb.query(query)
 
-data = { username: email,
+  data = []
-		 password: password }
+  res.each do |repo|
-
+    repo_name = repo['tags']['repo']
-res = HTTParty.post("https://hub.docker.com/v2/users/login/", body: data).body
+    count = repo['values'][0]['count']
-token = JSON.parse(res)["token"]
+    arch = repo['tags']['arch']
-
+    org = repo['tags']['org']
-headers = { token: token}
+    url = "https://hub.docker.com/r/#{org}/#{repo_name}"
-res = HTTParty.get(
+    data << { name: repo_name, count: count, arch: arch, org: org, url: url }
-  "https://hub.docker.com/v2/repositories/#{org}/",
-  query: { page_size: 100 },
-  headers: headers)
-
-data = JSON.parse(res.body)
-
-pull_counts = {}
-data["results"].each do |repo|
-	pull_counts[repo["name"]] = repo["pull_count"]
   end
 
-influxdb = InfluxDB::Client.new database: "dockerhub_stats"
+  data
-pull_counts.each do |repo, count|
+end
-	data = {
+
-	  values: { pull_count: count },
+set :logging, true
-	  tags:   { repo: repo }
+set :dump_errors, true
-	}
+set :bind, '0.0.0.0'
-
+
-	influxdb.write_point("dockerhub_stats", data)
+get '/stats' do
+  content_type :json
+  get_data.to_json
 end

app/stats_pull.rb

@@ -0,0 +1,93 @@
+require "httparty"
+require "awesome_print"
+require "influxdb"
+require "yaml"
+
+def processs_org(org)
+  arch = 'x86'
+  case(org)
+  when 'linuxserver' then
+    arch = 'x86'
+  when 'lsioarmhf' then
+    arch = 'armhf'
+  end
+
+  res = HTTParty.get(
+    "https://hub.docker.com/v2/repositories/#{org}/",
+    query: { page_size: 100   })
+
+  pull_counts = {}
+  continue = true
+  while continue
+    data = JSON.parse(res.body)
+    pull_counts.merge!(process_counts(data, arch))
+
+    if data.include?('next') && data['next'].nil? == false
+      url = data['next']
+      res = HTTParty.get(url)
+    else
+      continue = false
+    end
+
+  end
+
+  pull_counts
+end
+
+def process_counts(data, arch)
+  pull_counts = {}
+  data['results'].each do |repo|
+    name = repo['name']
+    count = repo['pull_count']
+    pull_counts["#{name}-#{arch}"] = { name: name, arch: arch, pull_count: count}
+  end
+
+  pull_counts
+end
+
+def insert_counts(data)
+  port = 8086
+  port = ENV['INFLUXDB_PORT'].to_i unless ENV['INFLUXDB_PORT'].nil?
+
+  influxdb = InfluxDB::Client.new database: "dockerhub_stats", port: port
+  data.each do |key, repo_data|
+    influx_data = {
+      values: { pull_count: repo_data[:pull_count] },
+      tags:   { repo: repo_data[:name], arch: repo_data[:arch], org: repo_data[:org] }
+    }
+
+    influxdb.write_point("dockerhub_stats", influx_data)
+  end
+
+end
+
+def run
+  orgs = nil
+
+  # pull orgs from env var
+  unless ENV['DOCKERHUB_ORGS'].nil?
+    orgs = ENV['DOCKERHUB_ORGS'].split(',')
+  end
+
+  # pull orgs from settings file if not specified in env var
+  if orgs.nil?
+    config_file = File.dirname(__FILE__) + "/config/settings.yml"
+    config = YAML.load_file(config_file)
+
+    orgs = config["orgs"]
+  end
+
+  counts = {}
+  orgs.each do |org|
+    org_counts = processs_org(org)
+    org_counts.each do |org_count|
+      org_count[1][:org] = org
+    end
+
+    counts.merge!(org_counts)
+  end
+
+  insert_counts(counts)
+end
+
+run

root/etc/cont-init.d/50-grafana

@@ -13,6 +13,7 @@ mkdir -p /config/etc/grafana /config/var/lib/grafana /config/var/log/grafana
 cp /etc/grafana/grafana.ini /config/etc/grafana
 
 /usr/sbin/grafana-server --homepath=/usr/share/grafana --config=/config/etc/grafana/grafana.ini cfg:default.paths.data=/config/var/lib/grafana cfg:default.paths.logs=/config/var/log/grafana 2>&1 &
+GRAFANA_PID=$!
 
 until nc -z -v -w5 localhost 3000 < /dev/null; do sleep 1; done
 
@@ -32,6 +33,5 @@ curl 'http://admin:admin@localhost:3000/api/datasources' \
 DATASOURCE
 echo
 
-kill `ps -ef | grep grafana-server | grep homepath | awk {'print $2'}`
+kill ${GRAFANA_PID}
-
 touch ${STAMP}

root/etc/cont-init.d/60-influxdb

@@ -15,11 +15,11 @@ cp /etc/influxdb/influxdb.conf /config/etc/influxdb/
 sed -i 's/\/var\/lib/\/config\/var\/lib/g' /config/etc/influxdb/influxdb.conf
 
 /usr/bin/influxd -pidfile /var/run/influxdb/influxd.pid -config /config/etc/influxdb/influxdb.conf 2>&1 &
+INFLUX_PID=$!
 
 # wait for influxdb to respond to requests
 until /usr/bin/influx -execute 'show databases'; do sleep 1; done
 /usr/bin/influx -execute 'create database dockerhub_stats'
 
-kill `ps -ef | grep influxd | grep pid | awk {'print $2'}`
+kill ${INFLUX_PID}
-
 touch ${STAMP}

root/etc/cont-init.d/70-dockerhub_stats

@@ -1,5 +1,9 @@
 #!/usr/bin/with-contenv bash
 
+if [ ! -d /config/app ]; then
+	mkdir /config/app
+fi
+
 if [ ! -f /config/app/settings.yml ]; then
 	cp /opt/dockerhub-stats/config/settings.yml.example /config/app/settings.yml
 fi
@@ -7,3 +11,5 @@ fi
 if [ ! -f /opt/dockerhub-stats/config/settings.yml ]; then
 	ln -s /config/app/settings.yml /opt/dockerhub-stats/config/settings.yml
 fi
+
+chmod 744 /etc/cron.d/dockerhub-stats

root/etc/cron.d/dockerhub-stats

@@ -1 +1 @@
-12 * * * * root /bin/bash -l -c 'cd /opt/dockerhub-stats; ruby stats.rb 2>&1'
+12 * * * * root /bin/bash -l -c 'cd /opt/dockerhub-stats; ruby stats_pull.rb 2>&1'

root/etc/services.d/cron/run

@@ -1,3 +1,3 @@
 #!/usr/bin/with-contenv bash
 
-/usr/sbin/cron
+/usr/sbin/cron -f

root/etc/services.d/sinatra/run

@@ -0,0 +1,5 @@
+#!/usr/bin/with-contenv bash
+
+cd /opt/dockerhub-stats/ || exit
+
+ruby stats.rb