用python把postgresql中的表内容复制到mysql中

postgresql中的表和mysql中的表结构不一样，过程中做点转换，特别是datetime的insert

还有activated 字段在原表中是布尔型，mysql表中是tinyint型

 

 

import psycopg2
import MySQLdb
import datetime, time


psycopg2_conn = psycopg2.connect(
        database="expat", 
        user="david", 
        password="1", 
        host="localhost", 
        port="5432"
    ) 
mysql_conn= MySQLdb.connect(
        host='localhost',
        port = 3306,
        user='root',
        passwd='1',
        db ='blog',
    )


def get_user_data():
    cur = psycopg2_conn.cursor()
    cur.execute("SELECT * FROM auth_user;")
    rows = cur.fetchall() 
    cur.close()
    psycopg2_conn.commit()
    psycopg2_conn.close()

    return rows
#get_user_data()


def save_user_to_mysql(rows):
    cur = mysql_conn.cursor()
    for row in rows:
        newrow = list()
        newrow.append(row[0]) # 0 id 
        newrow.append(row[4]) # 1 name
        newrow.append(row[7]) # 2 email
        newrow.append('') # 3 password
        newrow.append(1) if row[9] is True else newrow.append(0) # 5 activated
        newrow.append(0)      # 6 banned
        r2 = row[2]
        last_login = datetime.datetime(r2.year, r2.month, r2.day, r2.hour, r2.minute, r2.second)
        last_login = last_login.strftime('%Y-%m-%d %H:%M:%S')

        newrow.append(last_login) # 9 last_login
        newrow.append(0) # 12 protected
        r10 = row[10]
        created_at = datetime.datetime(r10.year, r10.month, r10.day, r10.hour, r10.minute, r10.second)
        created_at = created_at.strftime('%Y-%m-%d %H:%M:%S')
        newrow.append(created_at) #  13 created_at
        newrow.append(row[1]) # 16 password_old

        sql = '''
            insert into users(id, name, email, password, activated, banned,last_login,protected, created_at,password_old) 
                       values(%d, '%s', '%s',  '%s',     %s,        %d,    '%s',        %d,        '%s',        '%s')
        ''' % tuple(newrow)
        print sql
        cur.execute(sql) 

        if row[3] is True: # is_superuser is True
            cur.execute('''
                insert into users_groups (user_id, group_id)
                values(%d, 1)
                ''' % row[0]
                ) 

    cur.close()
    mysql_conn.commit()
    mysql_conn.close()
#save_user_to_mysql(None)



if __name__ == '__main__':
    rows=get_user_data()
    save_user_to_mysql(rows)
    pass

 

同时还要拷贝个profile表，稍微复杂点，sql写的很坑爹啊

import psycopg2
import MySQLdb
import datetime, time


psycopg2_conn = psycopg2.connect(
        database="expat", 
        user="david", 
        password="1", 
        host="localhost", 
        port="5432"
    ) 
mysql_conn= MySQLdb.connect(
        host='localhost',
        port = 3306,
        user='root',
        passwd='1',
        db ='blog',
    )


def get_user_data():
    cur = psycopg2_conn.cursor()
    #cur.execute("SELECT * FROM auth_user;")
    cur.execute(
    "select * from auth_user join accounts_myprofile on auth_user.id = accounts_myprofile.user_id;"
        )
    rows = cur.fetchall() 

    cur.close()
    psycopg2_conn.commit()
    psycopg2_conn.close()
    #for row in rows:
    #    print row
    return rows
#get_user_data()


def save_user_to_mysql(rows):
    cur = mysql_conn.cursor()
    for row in rows:
        ####### insert into user table
        userrow = list()
        userrow.append(row[0]) # 0 id 
        userrow.append(row[4]) # 1 name
        userrow.append(row[7]) # 2 email
        userrow.append('') # 3 password
        userrow.append(1) if row[9] is True else userrow.append(0) # 5 activated
        userrow.append(0)      # 6 banned
        r2 = row[2]
        last_login = datetime.datetime(r2.year, r2.month, r2.day, r2.hour, r2.minute, r2.second)
        last_login = last_login.strftime('%Y-%m-%d %H:%M:%S')

        userrow.append(last_login) # 9 last_login
        userrow.append(0) # 12 protected
        r10 = row[10]
        created_at = datetime.datetime(r10.year, r10.month, r10.day, r10.hour, r10.minute, r10.second)
        created_at = created_at.strftime('%Y-%m-%d %H:%M:%S')
        userrow.append(created_at) #  13 created_at
        userrow.append(row[1]) # 16 password_old

        sql = '''
            insert into users(id, name, email, password, activated, banned,last_login,protected, created_at,password_old) 
                       values(%d, '%s', '%s',  '%s',     %s,        %d,    '%s',        %d,        '%s',        '%s')
        ''' % tuple(userrow)
        
        cur.execute(sql) 

        ######## insert into user group
        if row[3] is True: # is_superuser is True
            cur.execute('''
                insert into users_groups (user_id, group_id)
                values(%d, 1)
                ''' % row[0]
                )

        ######## insert into profile table
        print row
        profile_row = list()
        profile_row.append(row[11])  # profile table field  id
        profile_row.append(row[0])   # user_id
        profile_row.append(row[5]) if row[5] else profile_row.append('')  # first_name
        profile_row.append(row[6]) if row[6] else profile_row.append('')  # last_name
        profile_row.append(row[16]) if row[16] else profile_row.append('')  # phone
        profile_row.append(row[15]) if row[15] else profile_row.append('')  # address
        profile_row.append(created_at)   # created_at
        profile_row.append(row[12]) if row[12] else profile_row.append('')  # mugshot
        profile_row.append(row[13])   # privacy
        profile_row.append(row[17]) if row[17] else profile_row.append('')  # company_name
        profile_row.append(row[14]) if row[14] else profile_row.append('null')   # housing_uid
        profile_row.append(row[18]) if row[18] else profile_row.append('null')   # live_area
        profile_row.append(row[19]) if row[19] else profile_row.append('null')   # come_from
        profile_row.append(row[20]) if row[20] else profile_row.append('null')   # come_reason
        profile_row.append(row[21]) if row[21] else profile_row.append('null')   # gender
        profile_row.append(row[22]) if row[22] else profile_row.append('null')   # marital
        profile_row.append(row[23]) if row[23] else profile_row.append('null')   # kids
        #profile_row.append(row[26])   # is_bussiness
        profile_row.append(1) if row[26] is True else profile_row.append(0) # is_bussiness
        #profile_row.append(row[27])   # is_paid_for_classifieds
        profile_row.append(1) if row[27] is True else profile_row.append(0) # is_paid_for_classifieds
        #profile_row.append(row[28])   # is_approved
        profile_row.append(1) if row[28] is True else profile_row.append(0) # is_approved
        if row[24]:
            r24 = row[24]
            birthday = datetime.datetime(r24.year, r24.month, r24.day)
            birthday = birthday.strftime('%Y-%m-%d')
            profile_row.append(birthday)   # birthday
        else:
            profile_row.append(row[24])



        profile_sql = '''
        insert into user_profile(id, user_id, first_name, last_name, phone, address,created_at,mugshot, privacy,company_name,housing_uid, live_area, come_from, come_reason, gender, marital, kids, is_bussiness, is_paid_for_classifieds, is_approved, birthday) 
                          values(%d, %d,      '%s',       '%s',      '%s',  '%s',   '%s',      '%s',    '%s',   '%s',        %s,          %s,        %s,        %s,          %s,     %s,      %s,   %s,            %d,                       %d,        '%s'  )
        '''% tuple(profile_row)
        print profile_sql
        cur.execute(profile_sql) 
    cur.close()
    mysql_conn.commit()
    mysql_conn.close()
#save_user_to_mysql(None)



if __name__ == '__main__':
    rows=get_user_data()
    save_user_to_mysql(rows)
    pass

 

 

 

article表 从pgsql拷到mysql

1  article表拷到本地pg数据库

   1.1 $ pg_dump -h xxx -U username expat -t article_article >/home/david/article_article.sql

   1.2 改sql文件中的用户名，改为本地数据库的用户名  例如username-->david

   1.3 $ psql -h localhost -U david postgres -t article_article< /home/david/article_article.sql

2  从postgres中导出数据到csv中

   2.1 新建一个csv文档，给权限，$ chmod 777 article_article.csv

   2.2 以超级用户postgres， 密码postgres 登录 pgsql, 

       运行sql, $ COPY (SELECT * FROM article_article) TO '/home/david/article_article.csv' WITH CSV HEADER;

   2.3 用vim把导出的csv文件的第一行删掉（culomn名删了）

3  csv中导入mysql

   3.1 mysql 是从 /var/lib/mysql/blog （blog是数据库名）这个目录下读文件的，所以把csv放到此目录下

       $ sudo cp article_article.csv /var/lib/mysql/blog/article_article.csv

   3.2 建一个表article_article 

   3.3然后在mysql运行sql命令：

load data infile 'article_article.csv' 

into table blog.article_article

fields terminated by ',' optionally enclosed by '"' escaped by '"' 

lines terminated by '\n'; 

4  pgsql用true false表示boolean，mysql用0,1，此时的mysql的article表中布尔值都是0，明显不对， 所以要做修改，我是用python脚本

import psycopg2
import MySQLdb
import datetime, time



psycopg2_conn = psycopg2.connect(
        #database="expat", 
        database="postgres", 
        user="david", 
        password="1", 
        host="localhost", 
        port="5432"
    )

mysql_conn= MySQLdb.connect(
        host='localhost',
        port = 3306,
        user='root',
        passwd='1',
        db ='blog',
    )
def import_mysql_boolean():
    '''

    '''
    cur = psycopg2_conn.cursor()
    cur.execute('''
        select id,is_approved, is_special, is_welcome, is_shf_featured,
            is_shf_sponsored, is_homepage_sponsored, is_home_featured
        from article_article 
        ''')
    rows = cur.fetchall() 
    #print rows

    cur.close()
    psycopg2_conn.commit()
    psycopg2_conn.close()

    cur2 = mysql_conn.cursor()

    for row in rows:
        arow = list()
        arow.append(row[0])
        arow.append(1) if row[1] is True else arow.append(0) # is_approved
        arow.append(1) if row[2] is True else arow.append(0) # is_special
        arow.append(1) if row[3] is True else arow.append(0) # is_welcome
        arow.append(1) if row[4] is True else arow.append(0) # is_shf_featured
        arow.append(1) if row[5] is True else arow.append(0) # is_shf_sponsored
        arow.append(1) if row[6] is True else arow.append(0) # is_homepage_sponsored
        arow.append(1) if row[7] is True else arow.append(0) # is_home_featured
        sql = '''
        update article_article set 
            is_approved={is_approved}, 
            is_special={is_special}, 
            is_welcome={is_welcome}, 
            is_shf_featured={is_shf_featured}, 
            is_shf_sponsored={is_shf_sponsored}, 
            is_homepage_sponsored={is_homepage_sponsored}, 
            is_home_featured={is_home_featured}
        where id = {id}
        '''.format(id=arow[0],
                   is_approved=arow[1],
                   is_special=arow[2],
                   is_welcome=arow[3],
                   is_shf_featured=arow[4],
                   is_shf_sponsored=arow[5],
                   is_homepage_sponsored=arow[6],
                   is_home_featured=arow[7],

            )

        #print sql
        cur2.execute(sql) 
    cur2.close()
    mysql_conn.commit()
    mysql_conn.close()


if __name__ == '__main__':
    #copy_category()
    #copy_author()
    #copy_article()
    import_mysql_boolean()
    #copy_article_tags()
    pass