java 链接mysql 产生500W数据模拟生成环境

java 插入数据到mysql 通过sqoop 导入到hive 中，kylin模拟见cube 时间和 数据膨胀率 kylin 数据插入到 HBase

Kylin

HBase 1.1.3

Hive 1.2.1

Hadoop 2.5.1

create table infoagetime(
prod_name char(10),
prod_id SMALLINT,
ods_date DATE
)

数据格式

oPmgBZxldW    75    2016-09-04
WSSAnnZrNy    57    2016-09-09

本人Java水平有限，大家自行改善，尤其是批量插入mysql语句

另工程中需要导入 mysql jar ：   mysql-connector-java-5.1.6.jar

V2

package com.wubaiwan.instmysql;import java.io.File;
import java.io.FileOutputStream;
import java.sql.DriverManager;
import java.util.Random;public class YiBaiWan {public static String getRandomString(int length) { // length表示生成字符串的长度String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";Random random = new Random();StringBuffer sb = new StringBuffer();for (int i = 0; i < length; i++) {int number = random.nextInt(base.length());sb.append(base.charAt(number));}return sb.toString();}public static final String url = "jdbc:mysql://192.168.184.168/hive2";public static final String name = "com.mysql.jdbc.Driver";public static final String user = "root";public static final String password = "123";public static java.sql.Connection conn = null;public static java.sql.PreparedStatement pst = null;static FileOutputStream out = null;public static void main(String args[]) {try {Class.forName(name);conn = DriverManager.getConnection(url, user, password);// 获取连接conn.setAutoCommit(false);} catch (Exception e1) {// TODO Auto-generated catch blocke1.printStackTrace();} // 指定连接类型Random random = new Random();// int k = random.nextInt();// System.out.println(k);int x = 0;try {out = new FileOutputStream(new File("D:/mysql.txt"));String sql = "insert into infoagetime(prod_name,prod_id,ods_date) values(?,?,?)";// System.out.println(sql);pst = conn.prepareStatement(sql);while (x < 5000000) {// System.out.println((int)(Math.random()*100));// System.out.println(getRandomString(10) + ',' +// (int)(Math.random()*100));// String k = getRandomString(10) + ',' +// (int)(Math.random()*100 )+ ','+ "2016-09-0" +// (int)(random.nextInt(9)%9 + 1)+"\r\n" ;// System.out.println(k);// out.write(k.getBytes());pst.setString(1, getRandomString(10));pst.setInt(2, (int) (Math.random() * 100));pst.setString(3, "2016-09-0" + (int) (random.nextInt(9) % 9 + 1));pst.addBatch();if (x % 1000 == 0) {pst.executeBatch();//pst.executeUpdate(arg0)}x++;}System.out.println("Commit");pst.executeBatch();conn.commit();out.close();conn.close();pst.close();} catch (Exception e) {// TODO Auto-generated catch blocke.printStackTrace();}finally{}}}

======================================

V1  这个版本会有java memory 异常

package com.wubaiwan.instmysql;import java.io.File;  
import java.io.FileOutputStream;
import java.sql.DriverManager;
import java.util.Random;public class YiBaiWan {  public static String getRandomString(int length) { //length表示生成字符串的长度      String base = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";         Random random = new Random();         StringBuffer sb = new StringBuffer();         for (int i = 0; i < length; i++) {         int number = random.nextInt(base.length());         sb.append(base.charAt(number));         }         return sb.toString();         }    public static final String url = "jdbc:mysql://192.168.184.168/hive2";    public static final String name = "com.mysql.jdbc.Driver";    public static final String user = "root";    public static final String password = "123";    public static java.sql.Connection conn = null;    public static java.sql.PreparedStatement pst = null;    public static void main(String args[]){  try {Class.forName(name);conn = DriverManager.getConnection(url, user, password);//获取连接 } catch (Exception e1) {// TODO Auto-generated catch blocke1.printStackTrace();}//指定连接类型   FileOutputStream out = null;     Random random = new Random();  //int k = random.nextInt();  //System.out.println(k);  int x = 0;  try {  out = new FileOutputStream(new File("D:/mysql.txt"));  while(x <5000000){  //System.out.println((int)(Math.random()*100));  //System.out.println(getRandomString(10) + ',' + (int)(Math.random()*100));  //String k = getRandomString(10) + ',' + (int)(Math.random()*100 )+ ','+ "2016-09-0" + (int)(random.nextInt(9)%9 + 1)+"\r\n" ;  //System.out.println(k);  //out.write(k.getBytes());  String sql = "insert into infoagetime(prod_name,prod_id,ods_date) values('"+getRandomString(10)+ "',"+(int)(Math.random()*100 )+","+ "'2016-09-0" + (int)(random.nextInt(9)%9 + 1) + "')"   ;//System.out.println(sql);pst = conn.prepareStatement(sql);  pst.executeUpdate();  if(x%10000 == 0 ) {System.out.println("Commit");conn.commit();}x++;  }   out.close();    conn.close();   pst.close();  }catch (Exception e) {  // TODO Auto-generated catch block  e.printStackTrace();  }  }  }