简单去除文本重复

<?php

error_reporting(0);

echo"/n"."输入要整理的文件:"."/n";

$dic=trim(fgets(STDIN));

$file=file($dic);

$array=preg_replace('/($/s*$)|(^/s*^)/m','',$file); //消除空行

$new_array=array_values(array_unique($array));//消除重复行

$new_filename="new_". basename($dic); www.zhishiwu.com

if(file_put_contents("$new_filename",join("/r/n",$new_array))){

        echo"------------------------------------------"."/r/n";

        echo"/n"."去除重复完毕!"."/r/n" ;

        echo"去除重复后的文件为:"."/r/n";

        echo dirname(__FILE__).DIRECTORY_SEPARATOR."$new_filename"."/r/n";

        echo"------------------------------------------"."/r/n";

}

else{ 

        echo"------------------------------------------"."/r/n";

        echo"错误!"."/r/n";

        echo"找不到文件!请检查输入路径是否存在!"."/r/n";

        echo"------------------------------------------"."/r/n";

exit();

}

?>

摘自 http://hi.baidu.com/nginxshel