Re: MMR: How do you tell that the MMR servers are in sync?

["Paul B. Henson" <henson@acm.org>]

On Tue, Jun 07, 2016 at 08:02:14AM -0400, Frank Swasey wrote:

> I am intending that only one will officially be active at a time. 
> However, I am doing that by activating the service IP addresses on the 
> system that I want to be active instead of using a load balancer.

Cool. In that case, here is the script we've been running for years to
check that our 4 MMR servers are synced. It occasionally blips when we
do a major idm run that creates or deletes 20-30 thousand accounts, but
other than that I rarely if ever get alerts from it. It runs on each of
the four servers, and our config management server populates it with the
names of the other three to check.

#! /usr/bin/perl

use Net::LDAP ();

my $other_csns = lookup_other_csns();
my $my_csns    = lookup_my_csns();

if (!compare_csns($my_csns, $other_csns)) {
	my $prev_other_csns = $other_csns;
	my $prev_my_csns    = $my_csns;

	sleep 120;

	$other_csns = lookup_other_csns();
	$my_csns    = lookup_my_csns();

	if (!compare_csns($my_csns, $other_csns)) {
		if ($prev_my_csns ne $my_csns) {
			if (!(date_check($my_csns, $other_csns))) {
				print STDERR "Warning: LDAP replica out of synchronization\n";
				print_csns($my_csns, $other_csns);
				exit(1);
			}
		}
		else {
			print STDERR "Error: LDAP replica out of syncronization, no apparent update progress seen\n";
			print_csns($my_csns, $other_csns);
			exit(1);
		}
	}
}

sub lookup_csns {
	my ($ldap, $server_name) = @_;

	my $search = $ldap->search(
		scope  => 'base',
		base   => "dc=cpp,dc=edu",
		filter => "(objectclass=*)",
		attrs  => [ 'contextCSN' ]
		);

	$search->code() and do {
		print STDERR "Error: failed to execute search on $server_name: "
			. $search->error() . " (" . $search->code() . ")\n";
		exit(1);
	};

	my $entry = $search->shift_entry() or do {
		print STDERR "Error: search on $server_name failed to find entry\n";
		exit(1);
	};

	my %csn_hash;

	foreach my $csn ($entry->get_value('contextCSN')) {
		defined($csn) or die("Error: no contextCSN attribute found in $server_name entry\n");
		if ($csn =~ m/Z#\d{6}#(\d{3})#.*/) {
			$csn_hash{$1} = $csn;
		}
	}

	my @csns;

	foreach my $key (sort(keys %csn_hash)) {
		push @csns, $csn_hash{$key};
	}

	my $csns_s = join(" ", @csns);
	return $csns_s;
}

sub lookup_other_csns {
	my @ret;

	foreach my $server ('ldap2','ldap3','ldap4') {
		my $ldap_master = Net::LDAP->new("$server.ldap.cpp.edu", timeout => 10) or do {
			print STDERR "Error: failed to connect to LDAP master $server: $@\n";
			print STDERR "Synchronization checks will not be performed on $server\n";
		};
		if (defined $ldap_master) {
			my $csns = lookup_csns($ldap_master, $server);
			push @ret, [$server, $csns];
			$ldap_master->unbind;
		}
	}

	return \@ret;
}

sub lookup_my_csns {
	my $ldap_slave = Net::LDAP->new("localhost", timeout => 10) or do {
		print STDERR "Error: failed to connect to local LDAP: $@\n";
		exit(1);
	};

	$csns = lookup_csns($ldap_slave, "localhost");
	$ldap_slave->unbind;
	return $csns;
}

sub compare_csns {
	my ($my_csns, $other_csns_list) = @_;
	my @other_csns = @$other_csns_list;

	foreach my $csnsref (@other_csns) {
		my ($servername, $csns) = @$csnsref;
		if (!($my_csns eq $csns)) {
			return 0;
		}
	}
	return 1;
}

sub date_check {
	my ($my_csns, $other_csns_list) = @_;

	my %my_csns_hash;

	foreach my $csn (split(" ", $my_csns)) {
		if ($csn =~ m/Z#\d{6}#(\d{3})/) {
			$my_csns_hash{$1} = $csn;
		}
	}

	my @other_csns = @$other_csns;

	foreach my $other_csnref (@other_csns) {
		my ($servername, $other_csn) = @$other_csnref;
		my %other_csns_hash;
		foreach my $csn (split(" ", $other_csn)) {
			if ($csn =~ m/Z#\d{6}#(\d{3})/) {
				$other_csns_hash{$1} = $csn;
			}
  		}
		foreach my $key (keys %my_csns_hash) {
			if (!($my_csns_hash{$key} eq $other_csns_hash{$key})) {
				# compare times.
				my $csn1    = $my_csns_hash{$key};
				my $csn2    = $other_csns_hash{$key};
				my $csn1_ts = $csn1; $csn1_date =~ s/Z.*//;
				my $csn2_ts = $csn2; $csn2_date =~ s/Z.*//;

				if ($csn2_ts - $csn1_ts > 1000) {
					return 0;
				}
			}
		}
	}
	return 1;
}

sub print_csns {
	my ($my_csns, $other_csns_list) = @_;
	my @other_csns = @$other_csns_list;

	print "My CSNs: \n";
	foreach my $csn (split(" ", $my_csns)) {
		print "    $csn\n";
	}
	foreach my $ocsnref (@other_csns) {
		my ($servername, $ocsn) = @$ocsnref;
		print "$servername CSNs: \n";
		foreach my $csn (split(" ", $ocsn)) {
			print "    $csn\n";
		}
	}
}