apps_plugin.c

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <sys/stat.h>

#include <sys/resource.h>
#include <sys/stat.h>

#include <errno.h>
#include <stdarg.h>
#include <locale.h>
#include <ctype.h>
#include <fcntl.h>

#include <malloc.h>
#include <dirent.h>
#include <arpa/inet.h>

#include <sys/types.h>
#include <pwd.h>
#include <grp.h>

#include "avl.h"

#include "common.h"
#include "log.h"
#include "procfile.h"
#include "../config.h"

#ifdef NETDATA_INTERNAL_CHECKS
#include <sys/prctl.h>
#endif

#define MAX_COMPARE_NAME 100
#define MAX_NAME 100
#define MAX_CMDLINE 1024

// the rates we are going to send to netdata
// will have this detail
// a value of:
// 1 will send just integer parts to netdata
// 100 will send 2 decimal points
// 1000 will send 3 decimal points
// etc.
#define RATES_DETAIL 10000ULL

int processors = 1;
pid_t pid_max = 32768;
int debug = 0;

int update_every = 1;
unsigned long long global_iterations_counter = 1;
unsigned long long file_counter = 0;
int proc_pid_cmdline_is_needed = 0;
int include_exited_childs = 1;
char *host_prefix = "";
char *config_dir = CONFIG_DIR;

pid_t *all_pids_sortlist = NULL;

// ----------------------------------------------------------------------------

void netdata_cleanup_and_exit(int ret) {
	exit(ret);
}


// ----------------------------------------------------------------------------
// system functions
// to retrieve settings of the system

long get_system_cpus(void) {
	procfile *ff = NULL;

	int processors = 0;

	char filename[FILENAME_MAX + 1];
	snprintfz(filename, FILENAME_MAX, "%s/proc/stat", host_prefix);

	ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
	if(!ff) return 1;

	ff = procfile_readall(ff);
	if(!ff) {
		procfile_close(ff);
		return 1;
	}

	unsigned int i;
	for(i = 0; i < procfile_lines(ff); i++) {
		if(!procfile_linewords(ff, i)) continue;

		if(strncmp(procfile_lineword(ff, i, 0), "cpu", 3) == 0) processors++;
	}
	processors--;
	if(processors < 1) processors = 1;

	procfile_close(ff);
	return processors;
}

pid_t get_system_pid_max(void) {
	procfile *ff = NULL;
	pid_t mpid = 32768;

	char filename[FILENAME_MAX + 1];
	snprintfz(filename, FILENAME_MAX, "%s/proc/sys/kernel/pid_max", host_prefix);
	ff = procfile_open(filename, NULL, PROCFILE_FLAG_DEFAULT);
	if(!ff) return mpid;

	ff = procfile_readall(ff);
	if(!ff) {
		procfile_close(ff);
		return mpid;
	}

	mpid = (pid_t)atoi(procfile_lineword(ff, 0, 0));
	if(!mpid) mpid = 32768;

	procfile_close(ff);
	return mpid;
}

// ----------------------------------------------------------------------------
// target
// target is the structure that process data are aggregated

struct target {
	char compare[MAX_COMPARE_NAME + 1];
	uint32_t comparehash;
	size_t comparelen;

	char id[MAX_NAME + 1];
	uint32_t idhash;

	char name[MAX_NAME + 1];

	uid_t uid;
	gid_t gid;

	unsigned long long minflt;
	unsigned long long cminflt;
	unsigned long long majflt;
	unsigned long long cmajflt;
	unsigned long long utime;
	unsigned long long stime;
	unsigned long long cutime;
	unsigned long long cstime;
	unsigned long long num_threads;
	unsigned long long rss;

	unsigned long long statm_size;
	unsigned long long statm_resident;
	unsigned long long statm_share;
	unsigned long long statm_text;
	unsigned long long statm_lib;
	unsigned long long statm_data;
	unsigned long long statm_dirty;

	unsigned long long io_logical_bytes_read;
	unsigned long long io_logical_bytes_written;
	unsigned long long io_read_calls;
	unsigned long long io_write_calls;
	unsigned long long io_storage_bytes_read;
	unsigned long long io_storage_bytes_written;
	unsigned long long io_cancelled_write_bytes;

	int *fds;
	unsigned long long openfiles;
	unsigned long long openpipes;
	unsigned long long opensockets;
	unsigned long long openinotifies;
	unsigned long long openeventfds;
	unsigned long long opentimerfds;
	unsigned long long opensignalfds;
	unsigned long long openeventpolls;
	unsigned long long openother;

	unsigned long processes;	// how many processes have been merged to this
	int exposed;				// if set, we have sent this to netdata
	int hidden;					// if set, we set the hidden flag on the dimension
	int debug;
	int ends_with;
	int starts_with;            // if set, the compare string matches only the
								// beginning of the command

	struct target *target;		// the one that will be reported to netdata
	struct target *next;
};


// ----------------------------------------------------------------------------
// apps_groups.conf
// aggregate all processes in groups, to have a limited number of dimensions

struct target *apps_groups_root_target = NULL;
struct target *apps_groups_default_target = NULL;
long apps_groups_targets = 0;

struct target *users_root_target = NULL;
struct target *groups_root_target = NULL;

struct target *get_users_target(uid_t uid)
{
	struct target *w;
	for(w = users_root_target ; w ; w = w->next)
		if(w->uid == uid) return w;

	w = calloc(sizeof(struct target), 1);
	if(unlikely(!w)) {
		error("Cannot allocate %lu bytes of memory", (unsigned long)sizeof(struct target));
		return NULL;
	}

	snprintfz(w->compare, MAX_COMPARE_NAME, "%u", uid);
	w->comparehash = simple_hash(w->compare);
	w->comparelen = strlen(w->compare);

	snprintfz(w->id, MAX_NAME, "%u", uid);
	w->idhash = simple_hash(w->id);

	struct passwd *pw = getpwuid(uid);
	if(!pw)
		snprintfz(w->name, MAX_NAME, "%u", uid);
	else
		snprintfz(w->name, MAX_NAME, "%s", pw->pw_name);

	netdata_fix_chart_name(w->name);

	w->uid = uid;

	w->next = users_root_target;
	users_root_target = w;

	if(unlikely(debug))
		fprintf(stderr, "apps.plugin: added uid %u ('%s') target\n", w->uid, w->name);

	return w;
}

struct target *get_groups_target(gid_t gid)
{
	struct target *w;
	for(w = groups_root_target ; w ; w = w->next)
		if(w->gid == gid) return w;

	w = calloc(sizeof(struct target), 1);
	if(unlikely(!w)) {
		error("Cannot allocate %lu bytes of memory", (unsigned long)sizeof(struct target));
		return NULL;
	}

	snprintfz(w->compare, MAX_COMPARE_NAME, "%u", gid);
	w->comparehash = simple_hash(w->compare);
	w->comparelen = strlen(w->compare);

	snprintfz(w->id, MAX_NAME, "%u", gid);
	w->idhash = simple_hash(w->id);

	struct group *gr = getgrgid(gid);
	if(!gr)
		snprintfz(w->name, MAX_NAME, "%u", gid);
	else
		snprintfz(w->name, MAX_NAME, "%s", gr->gr_name);

	netdata_fix_chart_name(w->name);

	w->gid = gid;

	w->next = groups_root_target;
	groups_root_target = w;

	if(unlikely(debug))
		fprintf(stderr, "apps.plugin: added gid %u ('%s') target\n", w->gid, w->name);

	return w;
}

// find or create a new target
// there are targets that are just aggregated to other target (the second argument)
struct target *get_apps_groups_target(const char *id, struct target *target)
{
	int tdebug = 0, thidden = 0, ends_with = 0;
	const char *nid = id;

	while(nid[0] == '-' || nid[0] == '+' || nid[0] == '*') {
		if(nid[0] == '-') thidden = 1;
		if(nid[0] == '+') tdebug = 1;
		if(nid[0] == '*') ends_with = 1;
		nid++;
	}
	uint32_t hash = simple_hash(id);

	struct target *w, *last = apps_groups_root_target;
	for(w = apps_groups_root_target ; w ; w = w->next) {
		if(w->idhash == hash && strncmp(nid, w->id, MAX_NAME) == 0)
			return w;

		last = w;
	}

	w = calloc(sizeof(struct target), 1);
	if(unlikely(!w)) {
		error("Cannot allocate %lu bytes of memory", (unsigned long)sizeof(struct target));
		return NULL;
	}

	strncpyz(w->id, nid, MAX_NAME);
	w->idhash = simple_hash(w->id);

	strncpyz(w->name, nid, MAX_NAME);

	strncpyz(w->compare, nid, MAX_COMPARE_NAME);
	int len = strlen(w->compare);
	if(w->compare[len - 1] == '*') {
		w->compare[len - 1] = '\0';
		w->starts_with = 1;
	}
	w->ends_with = ends_with;

	if(w->starts_with && w->ends_with)
		proc_pid_cmdline_is_needed = 1;

	w->comparehash = simple_hash(w->compare);
	w->comparelen = strlen(w->compare);

	w->hidden = thidden;
	w->debug = tdebug;
	w->target = target;

	// append it, to maintain the order in apps_groups.conf
	if(last) last->next = w;
	else apps_groups_root_target = w;

	if(unlikely(debug))
		fprintf(stderr, "apps.plugin: ADDING TARGET ID '%s', process name '%s' (%s), aggregated on target '%s', options: %s %s\n"
		        , w->id
				, w->compare, (w->starts_with && w->ends_with)?"substring":((w->starts_with)?"prefix":((w->ends_with)?"suffix":"exact"))
				, w->target?w->target->id:w->id
				, (w->hidden)?"hidden":"-"
				, (w->debug)?"debug":"-"
		);

	return w;
}

// read the apps_groups.conf file
int read_apps_groups_conf(const char *name)
{
	char filename[FILENAME_MAX + 1];

	snprintfz(filename, FILENAME_MAX, "%s/apps_%s.conf", config_dir, name);

	if(unlikely(debug))
		fprintf(stderr, "apps.plugin: process groups file: '%s'\n", filename);

	// ----------------------------------------

	procfile *ff = procfile_open(filename, " :\t", PROCFILE_FLAG_DEFAULT);
	if(!ff) return 1;

	procfile_set_quotes(ff, "'\"");

	ff = procfile_readall(ff);
	if(!ff) {
		procfile_close(ff);
		return 1;
	}

	unsigned long line, lines = procfile_lines(ff);

	for(line = 0; line < lines ;line++) {
		unsigned long word, words = procfile_linewords(ff, line);
		struct target *w = NULL;

		char *t = procfile_lineword(ff, line, 0);
		if(!t || !*t) continue;

		for(word = 0; word < words ;word++) {
			char *s = procfile_lineword(ff, line, word);
			if(!s || !*s) continue;
			if(*s == '#') break;

			if(t == s) continue;

			struct target *n = get_apps_groups_target(s, w);
			if(!n) {
				error("Cannot create target '%s' (line %lu, word %lu)", s, line, word);
				continue;
			}

			if(!w) w = n;
		}

		if(w) {
			int tdebug = 0, thidden = 0;

			while(t[0] == '-' || t[0] == '+') {
				if(t[0] == '-') thidden = 1;
				if(t[0] == '+') tdebug = 1;
				t++;
			}

			strncpyz(w->name, t, MAX_NAME);
			w->hidden = thidden;
			w->debug = tdebug;

			if(unlikely(debug))
				fprintf(stderr, "apps.plugin: AGGREGATION TARGET NAME '%s' on ID '%s', process name '%s' (%s), aggregated on target '%s', options: %s %s\n"
						, w->name
						, w->id
						, w->compare, (w->starts_with && w->ends_with)?"substring":((w->starts_with)?"prefix":((w->ends_with)?"suffix":"exact"))
						, w->target?w->target->id:w->id
						, (w->hidden)?"hidden":"-"
						, (w->debug)?"debug":"-"
				);
		}
	}

	procfile_close(ff);

	apps_groups_default_target = get_apps_groups_target("p+!o@w#e$i^r&7*5(-i)l-o_", NULL); // match nothing
	if(!apps_groups_default_target)
		error("Cannot create default target");
	else
		strncpyz(apps_groups_default_target->name, "other", MAX_NAME);

	return 0;
}


// ----------------------------------------------------------------------------
// data to store for each pid
// see: man proc

struct pid_stat {
	int32_t pid;
	char comm[MAX_COMPARE_NAME + 1];
	char cmdline[MAX_CMDLINE + 1];

	// char state;
	int32_t ppid;
	// int32_t pgrp;
	// int32_t session;
	// int32_t tty_nr;
	// int32_t tpgid;
	// uint64_t flags;

	// these are raw values collected
	unsigned long long minflt_raw;
	unsigned long long cminflt_raw;
	unsigned long long majflt_raw;
	unsigned long long cmajflt_raw;
	unsigned long long utime_raw;
	unsigned long long stime_raw;
	unsigned long long cutime_raw;
	unsigned long long cstime_raw;

	// these are rates
	unsigned long long minflt;
	unsigned long long cminflt;
	unsigned long long majflt;
	unsigned long long cmajflt;
	unsigned long long utime;
	unsigned long long stime;
	unsigned long long cutime;
	unsigned long long cstime;

	// int64_t priority;
	// int64_t nice;
	int32_t num_threads;
	// int64_t itrealvalue;
	// unsigned long long starttime;
	// unsigned long long vsize;
	unsigned long long rss;
	// unsigned long long rsslim;
	// unsigned long long starcode;
	// unsigned long long endcode;
	// unsigned long long startstack;
	// unsigned long long kstkesp;
	// unsigned long long kstkeip;
	// uint64_t signal;
	// uint64_t blocked;
	// uint64_t sigignore;
	// uint64_t sigcatch;
	// uint64_t wchan;
	// uint64_t nswap;
	// uint64_t cnswap;
	// int32_t exit_signal;
	// int32_t processor;
	// uint32_t rt_priority;
	// uint32_t policy;
	// unsigned long long delayacct_blkio_ticks;
	// uint64_t guest_time;
	// int64_t cguest_time;

	uid_t uid;
	gid_t gid;

	unsigned long long statm_size;
	unsigned long long statm_resident;
	unsigned long long statm_share;
	unsigned long long statm_text;
	unsigned long long statm_lib;
	unsigned long long statm_data;
	unsigned long long statm_dirty;

	unsigned long long io_logical_bytes_read_raw;
	unsigned long long io_logical_bytes_written_raw;
	unsigned long long io_read_calls_raw;
	unsigned long long io_write_calls_raw;
	unsigned long long io_storage_bytes_read_raw;
	unsigned long long io_storage_bytes_written_raw;
	unsigned long long io_cancelled_write_bytes_raw;

	unsigned long long io_logical_bytes_read;
	unsigned long long io_logical_bytes_written;
	unsigned long long io_read_calls;
	unsigned long long io_write_calls;
	unsigned long long io_storage_bytes_read;
	unsigned long long io_storage_bytes_written;
	unsigned long long io_cancelled_write_bytes;

	int *fds;						// array of fds it uses
	int fds_size;					// the size of the fds array

	int children_count;				// number of processes directly referencing this
	int keep;						// 1 when we need to keep this process in memory even after it exited
	int keeploops;					// increases by 1 every time keep is 1 and updated 0
	int updated;					// 1 when the process is currently running
	int merged;						// 1 when it has been merged to its parent
	int new_entry;					// 1 when this is a new process, just saw for the first time
	int read;						// 1 when we have already read this process for this iteration
	int sortlist;					// higher numbers = top on the process tree
									// each process gets a unique number

	struct target *target;			// app_groups.conf targets
	struct target *user_target;		// uid based targets
	struct target *group_target;	// gid based targets

	unsigned long long stat_collected_usec;
	unsigned long long last_stat_collected_usec;

	unsigned long long io_collected_usec;
	unsigned long long last_io_collected_usec;

	char *stat_filename;
	char *statm_filename;
	char *io_filename;
	char *cmdline_filename;

	struct pid_stat *parent;
	struct pid_stat *prev;
	struct pid_stat *next;
} *root_of_pids = NULL, **all_pids;

long all_pids_count = 0;

struct pid_stat *get_pid_entry(pid_t pid) {
	if(all_pids[pid]) {
		all_pids[pid]->new_entry = 0;
		return all_pids[pid];
	}

	all_pids[pid] = calloc(sizeof(struct pid_stat), 1);
	if(!all_pids[pid]) {
		error("Cannot allocate %zu bytes of memory", (size_t)sizeof(struct pid_stat));
		return NULL;
	}

	all_pids[pid]->fds = calloc(sizeof(int), 100);
	if(!all_pids[pid]->fds)
		error("Cannot allocate %zu bytes of memory", (size_t)(sizeof(int) * 100));
	else all_pids[pid]->fds_size = 100;

	if(root_of_pids) root_of_pids->prev = all_pids[pid];
	all_pids[pid]->next = root_of_pids;
	root_of_pids = all_pids[pid];

	all_pids[pid]->pid = pid;
	all_pids[pid]->new_entry = 1;

	all_pids_count++;

	return all_pids[pid];
}

void del_pid_entry(pid_t pid) {
	if(!all_pids[pid]) {
		error("attempted to free pid %d that is not allocated.", pid);
		return;
	}

	if(unlikely(debug))
		fprintf(stderr, "apps.plugin: process %d %s exited, deleting it.\n", pid, all_pids[pid]->comm);

	if(root_of_pids == all_pids[pid]) root_of_pids = all_pids[pid]->next;
	if(all_pids[pid]->next) all_pids[pid]->next->prev = all_pids[pid]->prev;
	if(all_pids[pid]->prev) all_pids[pid]->prev->next = all_pids[pid]->next;

	if(all_pids[pid]->fds) free(all_pids[pid]->fds);
	if(all_pids[pid]->stat_filename) free(all_pids[pid]->stat_filename);
	if(all_pids[pid]->statm_filename) free(all_pids[pid]->statm_filename);
	if(all_pids[pid]->io_filename) free(all_pids[pid]->io_filename);
	if(all_pids[pid]->cmdline_filename) free(all_pids[pid]->cmdline_filename);
	free(all_pids[pid]);

	all_pids[pid] = NULL;
	all_pids_count--;
}


// ----------------------------------------------------------------------------
// update pids from proc

int read_proc_pid_cmdline(struct pid_stat *p) {
	
	if(unlikely(!p->cmdline_filename)) {
		char filename[FILENAME_MAX + 1];
		snprintfz(filename, FILENAME_MAX, "%s/proc/%d/cmdline", host_prefix, p->pid);
		if(!(p->cmdline_filename = strdup(filename)))
			fatal("Cannot allocate memory for filename '%s'", filename);
	}

	int fd = open(p->cmdline_filename, O_RDONLY, 0666);
	if(unlikely(fd == -1)) goto cleanup;

	int i, bytes = read(fd, p->cmdline, MAX_CMDLINE);
	close(fd);

	if(unlikely(bytes <= 0)) goto cleanup;

	p->cmdline[bytes] = '\0';
	for(i = 0; i < bytes ; i++)
		if(unlikely(!p->cmdline[i])) p->cmdline[i] = ' ';

	if(unlikely(debug))
		fprintf(stderr, "Read file '%s' contents: %s\n", p->cmdline_filename, p->cmdline);

	return 0;

cleanup:
	// copy the command to the command line
	strncpyz(p->cmdline, p->comm, MAX_CMDLINE);
	return 0;
}

int read_proc_pid_ownership(struct pid_stat *p) {
	if(unlikely(!p->stat_filename)) {
		error("pid %d does not have a stat_filename", p->pid);
		return 1;
	}

	// ----------------------------------------
	// read uid and gid

	struct stat st;
	if(stat(p->stat_filename, &st) != 0) {
		error("Cannot stat file '%s'", p->stat_filename);
		return 1;
	}

	p->uid = st.st_uid;
	p->gid = st.st_gid;

	return 0;
}

int read_proc_pid_stat(struct pid_stat *p) {
	static procfile *ff = NULL;

	if(unlikely(!p->stat_filename)) {
		char filename[FILENAME_MAX + 1];
		snprintfz(filename, FILENAME_MAX, "%s/proc/%d/stat", host_prefix, p->pid);
		if(!(p->stat_filename = strdup(filename)))
			fatal("Cannot allocate memory for filename '%s'", filename);
	}

	int set_quotes = (!ff)?1:0;

	ff = procfile_reopen(ff, p->stat_filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
	if(unlikely(!ff)) goto cleanup;

	// if(set_quotes) procfile_set_quotes(ff, "()");
	if(set_quotes) procfile_set_open_close(ff, "(", ")");

	ff = procfile_readall(ff);
	if(unlikely(!ff)) goto cleanup;

	p->last_stat_collected_usec = p->stat_collected_usec;
	p->stat_collected_usec = timems();
	file_counter++;

	// parse the process name
	unsigned int i = 0;
	strncpyz(p->comm, procfile_lineword(ff, 0, 1), MAX_COMPARE_NAME);

	// p->pid			= atol(procfile_lineword(ff, 0, 0+i));
	// comm is at 1
	// p->state			= *(procfile_lineword(ff, 0, 2+i));
	p->ppid				= (int32_t) atol(procfile_lineword(ff, 0, 3 + i));
	// p->pgrp			= atol(procfile_lineword(ff, 0, 4+i));
	// p->session		= atol(procfile_lineword(ff, 0, 5+i));
	// p->tty_nr		= atol(procfile_lineword(ff, 0, 6+i));
	// p->tpgid			= atol(procfile_lineword(ff, 0, 7+i));
	// p->flags			= strtoull(procfile_lineword(ff, 0, 8+i), NULL, 10);

	unsigned long long last;

	last = p->minflt_raw;
	p->minflt_raw		= strtoull(procfile_lineword(ff, 0, 9+i), NULL, 10);
	p->minflt = (p->minflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->cminflt_raw;
	p->cminflt_raw		= strtoull(procfile_lineword(ff, 0, 10+i), NULL, 10);
	p->cminflt = (p->cminflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->majflt_raw;
	p->majflt_raw		= strtoull(procfile_lineword(ff, 0, 11+i), NULL, 10);
	p->majflt = (p->majflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->cmajflt_raw;
	p->cmajflt_raw		= strtoull(procfile_lineword(ff, 0, 12+i), NULL, 10);
	p->cmajflt = (p->cmajflt_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->utime_raw;
	p->utime_raw		= strtoull(procfile_lineword(ff, 0, 13+i), NULL, 10);
	p->utime = (p->utime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->stime_raw;
	p->stime_raw		= strtoull(procfile_lineword(ff, 0, 14+i), NULL, 10);
	p->stime = (p->stime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->cutime_raw;
	p->cutime_raw		= strtoull(procfile_lineword(ff, 0, 15+i), NULL, 10);
	p->cutime = (p->cutime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	last = p->cstime_raw;
	p->cstime_raw		= strtoull(procfile_lineword(ff, 0, 16+i), NULL, 10);
	p->cstime = (p->cstime_raw - last) * (1000000ULL * RATES_DETAIL) / (p->stat_collected_usec - p->last_stat_collected_usec);

	// p->priority		= strtoull(procfile_lineword(ff, 0, 17+i), NULL, 10);
	// p->nice			= strtoull(procfile_lineword(ff, 0, 18+i), NULL, 10);
	p->num_threads		= (int32_t) atol(procfile_lineword(ff, 0, 19 + i));
	// p->itrealvalue	= strtoull(procfile_lineword(ff, 0, 20+i), NULL, 10);
	// p->starttime		= strtoull(procfile_lineword(ff, 0, 21+i), NULL, 10);
	// p->vsize			= strtoull(procfile_lineword(ff, 0, 22+i), NULL, 10);
	p->rss				= strtoull(procfile_lineword(ff, 0, 23+i), NULL, 10);
	// p->rsslim		= strtoull(procfile_lineword(ff, 0, 24+i), NULL, 10);
	// p->starcode		= strtoull(procfile_lineword(ff, 0, 25+i), NULL, 10);
	// p->endcode		= strtoull(procfile_lineword(ff, 0, 26+i), NULL, 10);
	// p->startstack	= strtoull(procfile_lineword(ff, 0, 27+i), NULL, 10);
	// p->kstkesp		= strtoull(procfile_lineword(ff, 0, 28+i), NULL, 10);
	// p->kstkeip		= strtoull(procfile_lineword(ff, 0, 29+i), NULL, 10);
	// p->signal		= strtoull(procfile_lineword(ff, 0, 30+i), NULL, 10);
	// p->blocked		= strtoull(procfile_lineword(ff, 0, 31+i), NULL, 10);
	// p->sigignore		= strtoull(procfile_lineword(ff, 0, 32+i), NULL, 10);
	// p->sigcatch		= strtoull(procfile_lineword(ff, 0, 33+i), NULL, 10);
	// p->wchan			= strtoull(procfile_lineword(ff, 0, 34+i), NULL, 10);
	// p->nswap			= strtoull(procfile_lineword(ff, 0, 35+i), NULL, 10);
	// p->cnswap		= strtoull(procfile_lineword(ff, 0, 36+i), NULL, 10);
	// p->exit_signal	= atol(procfile_lineword(ff, 0, 37+i));
	// p->processor		= atol(procfile_lineword(ff, 0, 38+i));
	// p->rt_priority	= strtoul(procfile_lineword(ff, 0, 39+i), NULL, 10);
	// p->policy		= strtoul(procfile_lineword(ff, 0, 40+i), NULL, 10);
	// p->delayacct_blkio_ticks		= strtoull(procfile_lineword(ff, 0, 41+i), NULL, 10);
	// p->guest_time	= strtoull(procfile_lineword(ff, 0, 42+i), NULL, 10);
	// p->cguest_time	= strtoull(procfile_lineword(ff, 0, 43), NULL, 10);

	if(unlikely(debug || (p->target && p->target->debug)))
		fprintf(stderr, "apps.plugin: READ PROC/PID/STAT: %s/proc/%d/stat, process: '%s' on target '%s' (dt=%llu) VALUES: utime=%llu, stime=%llu, cutime=%llu, cstime=%llu, minflt=%llu, majflt=%llu, cminflt=%llu, cmajflt=%llu, threads=%d\n", host_prefix, p->pid, p->comm, (p->target)?p->target->name:"UNSET", p->stat_collected_usec - p->last_stat_collected_usec, p->utime, p->stime, p->cutime, p->cstime, p->minflt, p->majflt, p->cminflt, p->cmajflt, p->num_threads);

	if(unlikely(global_iterations_counter == 1)) {
		p->minflt			= 0;
		p->cminflt			= 0;
		p->majflt			= 0;
		p->cmajflt			= 0;
		p->utime			= 0;
		p->stime			= 0;
		p->cutime			= 0;
		p->cstime			= 0;
	}

	return 0;

cleanup:
	p->minflt			= 0;
	p->cminflt			= 0;
	p->majflt			= 0;
	p->cmajflt			= 0;
	p->utime			= 0;
	p->stime			= 0;
	p->cutime			= 0;
	p->cstime			= 0;
	p->num_threads		= 0;
	p->rss				= 0;
	return 1;
}

int read_proc_pid_statm(struct pid_stat *p) {
	static procfile *ff = NULL;

	if(unlikely(!p->statm_filename)) {
		char filename[FILENAME_MAX + 1];
		snprintfz(filename, FILENAME_MAX, "%s/proc/%d/statm", host_prefix, p->pid);
		if(!(p->statm_filename = strdup(filename)))
			fatal("Cannot allocate memory for filename '%s'", filename);
	}

	ff = procfile_reopen(ff, p->statm_filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
	if(unlikely(!ff)) goto cleanup;

	ff = procfile_readall(ff);
	if(unlikely(!ff)) goto cleanup;

	file_counter++;

	p->statm_size			= strtoull(procfile_lineword(ff, 0, 0), NULL, 10);
	p->statm_resident		= strtoull(procfile_lineword(ff, 0, 1), NULL, 10);
	p->statm_share			= strtoull(procfile_lineword(ff, 0, 2), NULL, 10);
	p->statm_text			= strtoull(procfile_lineword(ff, 0, 3), NULL, 10);
	p->statm_lib			= strtoull(procfile_lineword(ff, 0, 4), NULL, 10);
	p->statm_data			= strtoull(procfile_lineword(ff, 0, 5), NULL, 10);
	p->statm_dirty			= strtoull(procfile_lineword(ff, 0, 6), NULL, 10);

	return 0;

cleanup:
	p->statm_size			= 0;
	p->statm_resident		= 0;
	p->statm_share			= 0;
	p->statm_text			= 0;
	p->statm_lib			= 0;
	p->statm_data			= 0;
	p->statm_dirty			= 0;
	return 1;
}

int read_proc_pid_io(struct pid_stat *p) {
	static procfile *ff = NULL;

	if(unlikely(!p->io_filename)) {
		char filename[FILENAME_MAX + 1];
		snprintfz(filename, FILENAME_MAX, "%s/proc/%d/io", host_prefix, p->pid);
		if(!(p->io_filename = strdup(filename)))
			fatal("Cannot allocate memory for filename '%s'", filename);
	}

	// open the file
	ff = procfile_reopen(ff, p->io_filename, NULL, PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
	if(unlikely(!ff)) goto cleanup;

	ff = procfile_readall(ff);
	if(unlikely(!ff)) goto cleanup;

	file_counter++;

	p->last_io_collected_usec = p->io_collected_usec;
	p->io_collected_usec = timems();

	unsigned long long last;

	last = p->io_logical_bytes_read_raw;
	p->io_logical_bytes_read_raw = strtoull(procfile_lineword(ff, 0, 1), NULL, 10);
	p->io_logical_bytes_read = (p->io_logical_bytes_read_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	last = p->io_logical_bytes_written_raw;
	p->io_logical_bytes_written_raw = strtoull(procfile_lineword(ff, 1, 1), NULL, 10);
	p->io_logical_bytes_written = (p->io_logical_bytes_written_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	last = p->io_read_calls_raw;
	p->io_read_calls_raw = strtoull(procfile_lineword(ff, 2, 1), NULL, 10);
	p->io_read_calls = (p->io_read_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	last = p->io_write_calls_raw;
	p->io_write_calls_raw = strtoull(procfile_lineword(ff, 3, 1), NULL, 10);
	p->io_write_calls = (p->io_write_calls_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	last = p->io_storage_bytes_read_raw;
	p->io_storage_bytes_read_raw = strtoull(procfile_lineword(ff, 4, 1), NULL, 10);
	p->io_storage_bytes_read = (p->io_storage_bytes_read_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	last = p->io_storage_bytes_written_raw;
	p->io_storage_bytes_written_raw = strtoull(procfile_lineword(ff, 5, 1), NULL, 10);
	p->io_storage_bytes_written = (p->io_storage_bytes_written_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	last = p->io_cancelled_write_bytes_raw;
	p->io_cancelled_write_bytes_raw = strtoull(procfile_lineword(ff, 6, 1), NULL, 10);
	p->io_cancelled_write_bytes = (p->io_cancelled_write_bytes_raw - last) * (1000000ULL * RATES_DETAIL) / (p->io_collected_usec - p->last_io_collected_usec);

	if(unlikely(global_iterations_counter == 1)) {
		p->io_logical_bytes_read 		= 0;
		p->io_logical_bytes_written 	= 0;
		p->io_read_calls 				= 0;
		p->io_write_calls 				= 0;
		p->io_storage_bytes_read 		= 0;
		p->io_storage_bytes_written 	= 0;
		p->io_cancelled_write_bytes		= 0;
	}

	return 0;

cleanup:
	p->io_logical_bytes_read 		= 0;
	p->io_logical_bytes_written 	= 0;
	p->io_read_calls 				= 0;
	p->io_write_calls 				= 0;
	p->io_storage_bytes_read 		= 0;
	p->io_storage_bytes_written 	= 0;
	p->io_cancelled_write_bytes		= 0;
	return 1;
}

unsigned long long global_utime = 0;
unsigned long long global_stime = 0;

int read_proc_stat() {
	static char filename[FILENAME_MAX + 1] = "";
	static procfile *ff = NULL;
	static unsigned long long utime_raw = 0, stime_raw = 0, ntime_raw = 0, collected_usec = 0, last_collected_usec = 0;

	if(unlikely(!ff)) {
		snprintfz(filename, FILENAME_MAX, "%s/proc/stat", host_prefix);
		ff = procfile_open(filename, " \t:", PROCFILE_FLAG_DEFAULT);
		if(unlikely(!ff)) goto cleanup;
	}

	ff = procfile_readall(ff);
	if(unlikely(!ff)) goto cleanup;

	last_collected_usec = collected_usec;
	collected_usec = timems();

	file_counter++;

	unsigned long long last;

	last = utime_raw;
	utime_raw = strtoull(procfile_lineword(ff, 0, 1), NULL, 10);
	global_utime = (utime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec);

	last = ntime_raw;
	ntime_raw = strtoull(procfile_lineword(ff, 0, 2), NULL, 10);
	global_utime += (ntime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec);

	last = stime_raw;
	stime_raw = strtoull(procfile_lineword(ff, 0, 3), NULL, 10);
	global_stime = (stime_raw - last) * (1000000ULL * RATES_DETAIL) / (collected_usec - last_collected_usec);

	if(unlikely(global_iterations_counter == 1)) {
		global_utime = 0;
		global_stime = 0;
	}

	return 0;

cleanup:
	global_utime = 0;
	global_stime = 0;
	return 1;
}


// ----------------------------------------------------------------------------
// file descriptor
// this is used to keep a global list of all open files of the system
// it is needed in order to calculate the unique files processes have open

#define FILE_DESCRIPTORS_INCREASE_STEP 100

struct file_descriptor {
	avl avl;
#ifdef NETDATA_INTERNAL_CHECKS
	uint32_t magic;
#endif /* NETDATA_INTERNAL_CHECKS */
	uint32_t hash;
	const char *name;
	int type;
	int count;
	int pos;
} *all_files = NULL;

int all_files_len = 0;
int all_files_size = 0;

int file_descriptor_compare(void* a, void* b) {
#ifdef NETDATA_INTERNAL_CHECKS
	if(((struct file_descriptor *)a)->magic != 0x0BADCAFE || ((struct file_descriptor *)b)->magic != 0x0BADCAFE)
		error("Corrupted index data detected. Please report this.");
#endif /* NETDATA_INTERNAL_CHECKS */

	if(((struct file_descriptor *)a)->hash < ((struct file_descriptor *)b)->hash)