percona sync simplified to run over huge database

if [ $# -lt 3 ] ; then
echo "USAGE $0 slavename db_name type"
echo "slavename [e.g db2, db3, db4 ]"
echo "db_name [e.g db1 db2 etc... calm down one at a time]"
echo "type [xxl_table_array xl_table_array unwanted_table all_small]"

if [ -e "${LOCKFILE}" ]; then
echo "Already running."
exit 99


echo $! > "${LOCKFILE}"
chmod 644 "${LOCKFILE}"
backup_dir="./$1-table-sync-`date +%Y-%m-%d-%H-%M`"
mkdir -p $backup_dir


case "$table_size" in
tbls="xl_table_1 xl_table_2"
tbls="large_table_1 large_table_2"
tbls="temp_table1 temp_table2"
echo "chow chow; did you miss anything?"

for tb in $tbls; do
pt-table-sync h=db1,D=$db,t=$tb h=$slavehost --user=username --password=password --no-check-slave --print >> $backup_dir/$db-$tb.log

tbls="$(mysql -Bse "use $db; show tables;" | grep -v 'all_but_shit_tables\|all_but_shit_table_2')"
for tb in $tbls; do
pt-table-sync h=db1,D=$db,t=$tb h=$slavehost --user=username --password=password --no-check-slave --print >> $backup_dir/$db-$tb.log


get_list_of_tables $2
rm -f "${LOCKFILE}"


docker basics

#How to install Docker in centos 6.5
yum install docker-io

#Lets start service now
service docker start

#ensure its in boot
chkconfig docker on

#Explore other commands
docker help

# Pull image from docker repository
docker pull centos

# Lets run bash from container.
docker run -i -t centos /bin/bash

#Show me current running docker containers
docker ps

#Show me all container
docker ps -a

#show me latest container
docker ps -l

#Attach to detached container using container ID
docker attach c3852139b652

Could not evaluate: Cannot allocate memory [puppet]

Error: /Stage[main]/Vsftpd/Service[vsftpd]: Could not evaluate: Cannot allocate memory – fork(2)
Error: /Stage[main]/Graphite::Install/Package[whisper]: Could not evaluate: Cannot allocate memory – /usr/bin/pip freeze 2>&1
Error: /Stage[main]/Myclass/Exec[myexec]/returns: change from notrun to 0 failed: Cannot allocate memory – fork(2)

memory issue:

# Modify this if you’d like to change the memory allocation, enable JMX, etc
JAVA_ARGS=”-Xms512m -Xmx512m -XX:MaxPermSize=256m”

#Or get more memory 😀

AWS Automated Snapshot


# Volumelist => volume_id:someidentifiername:retaintion_period


snapshot_volumes() {
for volume in $volume_list; do

volume_id=$(echo "$volume" | cut -d':' -f1)
host_name=$(echo "$volume" | cut -d':' -f2)
retention_days=$(echo "$volume" | cut -d':' -f3)
retention_date_in_seconds=$(date +%s --date "$retention_days days ago")
snapshot_description=$(date +%Y-%m-%d)

snapshot_id=$(aws ec2 create-snapshot --output=text --description $host_name-$snapshot_description --volume-id $volume_id --query SnapshotId)
aws ec2 create-tags --resource $snapshot_id --tags Key=CreatedBy,Value=AutomatedBackup


cleanup_snapshots() {
for volume in $volume_list; do

volume_id=$(echo "$volume" | cut -d':' -f1)
host_name=$(echo "$volume" | cut -d':' -f2)
retention_days=$(echo "$volume" | cut -d':' -f3)
retention_date_in_seconds=$(date +%s --date "$retention_days days ago")

snapshot_list=$(aws ec2 describe-snapshots --output=text --filters "Name=volume-id,Values=$volume_id" "Name=tag:CreatedBy,Values=AutomatedBackup" --query Snapshots[].SnapshotId)
for snapshot in $snapshot_list; do

snapshot_date=$(aws ec2 describe-snapshots --output=text --snapshot-ids $snapshot --query Snapshots[].StartTime | awk -F "T" '{printf "%s\n", $1}')
snapshot_date_in_seconds=$(date "--date=$snapshot_date" +%s)
snapshot_description=$(aws ec2 describe-snapshots --snapshot-id $snapshot --query Snapshots[].Description)

if (( $snapshot_date_in_seconds <= $retention_date_in_seconds )); then aws ec2 delete-snapshot --snapshot-id $snapshot echo "aws ec2 delete-snapshot --snapshot-id $snapshot" else echo "Nothing to delete $snapshot" fi done done } cleanup_snapshots snapshot_volumes

git basics

# Install git server
yum install git

# Lets use user called git [specify home directory and group if required]
useradd git

# Into users home directory
cd /home/git/

# Initiate bare repo
git init --bare --shared


GIT Revert back to a previous commit/s

#Find the git commit code you want to revert back to
git log

commit bed35e87a07ae81f795598acfde163acb184211c
Author: root 
Date:   Mon Jun 6 12:08:19 2012 +0100

# Now reset
git reset bed35e87a07ae81f795598acfde163acb184211c

# Now move pointer to previous HEAD
git reset --soft HEAD@{1}

# Commit the changes
git commit -m "reverted to bed35e87a07ae81f795598acfde163acb184211c"

#Update working copy, this reflects new commit
git reset --hard


# Push changes
git push origin master

Above method is good if you want to jump back to any previous commit, however to jump back to immediate commit, you can

git reset --hard HEAD

virtualenv with centos

Download Python and extract it
xz -d Python-2.7.8.tar.xz
tar -xvf Python-2.7.8.tar

Installation process
Since we already installed all the dependencies we are ready to go:

# Enter the directory:
cd Python-2.7.8

# Run the configure:
./configure --prefix=/usr/local

# compile and install it:
make altinstall

# Checking Python version:
[root@tikejhya ~]# python2.7 -V
Python 2.7.8
If you need set PATH variable check the line below:

export PATH=”/usr/local/bin:$PATH”
Installing pip and virtualenv
Now we have Python installed, but something is missing isn’t? Yes! We need pip and virtualenv.

Install setuptools
wget --no-check-certificate

# Extract the files:
tar -xvf setuptools-1.4.2.tar.gz
cd setuptools-1.4.2

# Install setuptools using the Python 2.7.8:
python2.7 install
Install pip
curl | python2.7 -
And finally virtualenv
pip2.7 install virtualenv

virtualenv projectname
source projectname/bin/activate

python --version

#exit virtualenv

#python binary location

linux spf with postfix

#spf with postfix

yum --enablerepo=epel install python-dns python-pydns

wget [latest today]
tar -zxvf pyspf-2.0.12t.tar.gz
cd pyspf-2.0.12/
python build
python install

wget [latest today]
tar -zxvf pypolicyd-spf-1.3.2.tar.gz
cd pypolicyd-spf-1.3.2/
python build
python install

policyd-spf unix - n n - 0 spawn
user=nobody argv=/usr/bin/policyd-spf

smtpd_recipient_restrictions = reject_unauth_destination, check_policy_service unix:private/policyd-spf

try executing /usr/bin/policyd-spf

/usr/bin/python /usr/bin/policyd-spf /etc/postfix-policyd-spf-python/policyd-spf.conf

odbc in centos

# centos
yum install mysql-connector-odbc

apt-get install libmyodbc

root@ip-10-0-1-82:~# odbcinst -q -s
root@ip-10-0-1-82:~# odbcinst -q -d
root@ip-10-0-1-82:~# odbcinst -j
unixODBC 2.2.14
DRIVERS…………: /etc/odbcinst.ini
SYSTEM DATA SOURCES: /etc/odbc.ini
USER DATA SOURCES..: /root/.odbc.ini
SQLULEN Size…….: 8
SQLLEN Size……..: 8
root@ip-10-0-1-82:~# isql -S testdb -U root -P somepassword

checking shared odbc libraries linked to isql for default directories…
strings: ”: No such file
trying /tmp/sql … no
trying /tmp/sql … no
trying /etc … OK
checking odbc.ini files
reading /root/.odbc.ini
[testdb] found in /root/.odbc.ini
found this section:
Driver = /usr/lib/x86_64-linux-gnu/odbc/
SERVER = localhost
USER = root
PASSWORD = somepassword
PORT = 3306
Charset = UTF8
looking for driver for DSN [testdb] in /root/.odbc.ini
found driver line: ” Driver = /usr/lib/x86_64-linux-gnu/odbc/”
driver “/usr/lib/x86_64-linux-gnu/odbc/” found for [testdb] in .odbc.ini
found driver named “/usr/lib/x86_64-linux-gnu/odbc/”
/usr/lib/x86_64-linux-gnu/odbc/ is an executable file
“SERVER” found, not using freetds.conf
SERVER is “localhost”

Configuration looks OK. Connection details:

DSN: testdb
odbc.ini: /root/.odbc.ini
Driver: /usr/lib/x86_64-linux-gnu/odbc/
Server hostname: localhost

Attempting connection as root …
+ isql testdb root somepassword -v
| Connected! |
| |
| sql-statement |
| help [tablename] |
| quit |
| |

adding new volume ec2 or expanding disk

fdisk -l
file -s /dev/xvdf
df -h
mkfs -t ext4 /dev/xvdf
mkdir -p /testdata
mount /dev/xvdf /testdata

#Expanding same disk
Stop the instance
Create a snapshot from the volume
Create a new volume based on the snapshot increasing the size
Check and remember the current’s volume mount point (i.e. /dev/sda1)
Detach current volume
Attach the recently created volume to the instance, setting the exact mount point
Restart the instance
Access via SSH to the instance and run fdisk /dev/xvde

WARNING: DOS-compatible mode is deprecated. It’s strongly recommended to switch off the mode (command ‘c’) and change display units to sectors (command ‘u’)
Hit p to show current partitions
Hit d to delete current partitions (if there are more than one, you have to delete one at a time) NOTE: Don’t worry data is not lost
Hit n to create a new partition
Hit p to set it as primary
Hit 1 to set the first cylinder
Set the desired new space (if empty the whole space is reserved)
Hit a to make it bootable
Hit 1 and w to write changes
Reboot instance
Log via SSH and run resize2fs /dev/xvde1
Finally check the new space running df -h

diff between two files

diff -u file1 file2 | grep -E “^\+”