performance - MongoDB Slow query by ID -
i migrated mongodb database windows server centos. version 2.4.9. noticed have slow retrieval of records _id field! ran repair database on weekend bu did not solve problem. have method retrieves records ids (with in operator) (using spring data mongodb 1.4.1.release):
@override public map<string, record> findasmapids( final string[] ids, final componenttype... comps ) { if( null == ids || 0 == ids.length ) { return null; } map<string,record> result = new hashmap<string,record>(); final criteria cr = where("_id").in( idarrfunction.apply(ids) ); final query qry = new query( cr ); setfieldstoreturn( qry, comps ); long start = system.currenttimemillis(); list<record> ritems = gettemplate().find(qry, record.class); long end = system.currenttimemillis(); system.out.println( "findasmapids()::" + (end-start) ); for( record r: ritems ) { result.put( r.getid(), r ); } return result; } public mongooperations gettemplate() { return template; } the function idarrfunction returns array of objectid[]
you notice i'm timing data retrieval call. i'm not passing more 10 ids @ time. here timing output i'm getting in logs when call method several times:
findasmapids()::2092 findasmapids()::6666 findasmapids()::10007 findasmapids()::15132 findasmapids()::18341 findasmapids()::24709 findasmapids()::28731 findasmapids()::33888 findasmapids()::36862 findasmapids()::40206 you notice time grows each call made method.
here collection stats:
> db.record.stats(); { "ns" : "mydb.record", "count" : 1491427, "size" : 18292198304, "avgobjsize" : 12264.896843090544, "storagesize" : 18899890096, "numextents" : 30, "nindexes" : 13, "lastextentsize" : 2146426864, "paddingfactor" : 1, "systemflags" : 1, "userflags" : 0, "totalindexsize" : 734401024, "indexsizes" : { "_id_" : 43561728, "orgid" : 63192304, "type" : 51018240, "orghierarchy" : 63274064, "orghierarchy_mrn" : 75603472, "dos" : 25525472, "org_pdt" : 77189616, "orghierarchy_processeddttm" : 77287728, "processeddttm_-1" : 37536016, "orgid_states" : 68367712, "states" : 27553120, "states_1_processeddttm_-1" : 41624016, "orgid_1_states_1_processeddttm_-1" : 82667536 }, "ok" : 1 } if printout query object java this:
query: { "_id" : { "$in" : [ { "$oid" : "51cbe96dcb81c43374ca9516"} , { "$oid" : "51cbe892cb81c43374ca5e48"} , { "$oid" : "51cbe910cb81c43374ca7aba"} , { "$oid" : "51c00739cb81e6b83ad29d14"} , { "$oid" : "51c06ff4cb81e6b83ad4f69a"} , { "$oid" : "51c07010cb81e6b83ad4f74e"} , { "$oid" : "51c06da5cb81e6b83ad4e775"} , { "$oid" : "51c081bbcb81e6b83ad56437"} , { "$oid" : "51c07fe2cb81e6b83ad557d2"} , { "$oid" : "51c081bbcb81e6b83ad56433"}]}}, fields: { "text" : 1 , "type" : 1 , "header" : 1 , "orgid" : 1 , "vitals" : 1 , "_id" : 1 , "processeddttm" : 1 , "orghierarchy" : 1 }, sort: null also if run explain:
> db.record.find({ "_id" : { "$in" : [ objectid("51ba0b43cb81e6b83acd9e6b") , objectid("51a8feb5cb8168350e373428") , objectid("51a90053cb8168350e373948") , objectid("51a901fecb8168350e373e82") , objectid("51a8f3e9cb8168350e3711d1") , objectid("51a8f3eacb8168350e3711d2") , objectid("51a8ff3ecb8168350e3735df") , objectid("51a8f59ecb8168350e37172b") , objectid("51a8f9aacb8168350e372418") , objectid("51a8f786cb8168350e371d42")]}}).explain(); { "cursor" : "btreecursor _id_ multi", "ismultikey" : false, "n" : 10, "nscannedobjects" : 10, "nscanned" : 18, "nscannedobjectsallplans" : 10, "nscannedallplans" : 18, "scanandorder" : false, "indexonly" : false, "nyields" : 0, "nchunkskips" : 0, "millis" : 0, "indexbounds" : { "_id" : [ [ objectid("51a8f3e9cb8168350e3711d1"), objectid("51a8f3e9cb8168350e3711d1") ], [ objectid("51a8f3eacb8168350e3711d2"), objectid("51a8f3eacb8168350e3711d2") ], [ objectid("51a8f59ecb8168350e37172b"), objectid("51a8f59ecb8168350e37172b") ], [ objectid("51a8f786cb8168350e371d42"), objectid("51a8f786cb8168350e371d42") ], [ objectid("51a8f9aacb8168350e372418"), objectid("51a8f9aacb8168350e372418") ], [ objectid("51a8feb5cb8168350e373428"), objectid("51a8feb5cb8168350e373428") ], [ objectid("51a8ff3ecb8168350e3735df"), objectid("51a8ff3ecb8168350e3735df") ], [ objectid("51a90053cb8168350e373948"), objectid("51a90053cb8168350e373948") ], [ objectid("51a901fecb8168350e373e82"), objectid("51a901fecb8168350e373e82") ], [ objectid("51ba0b43cb81e6b83acd9e6b"), objectid("51ba0b43cb81e6b83acd9e6b") ] ] }, "server" : "ip-10-0-0-41:27017" } i turned on profiler , get:
{ "op":"query", "ns":"sytrue.record", "query":{ "_id":{ "$in":[ objectid("51b1c98dcb81e4c1566b7436"), objectid("51b1cb3ccb81e4c1566b79ab"), objectid("51b1d741cb81e4c1566ba219"), objectid("51b1caa8cb81e4c1566b77d4"), objectid("51b1c98ccb81e4c1566b7433"), objectid("51b1c691cb81e4c1566b6a8c"), objectid("51b1cc21cb81e4c1566b7c9f"), objectid("51b1c390cb81e4c1566b60c9"), objectid("51b1d578cb81e4c1566b9c1d"), objectid("51b1e13dcb81e4c1566bc30e") ] } }, "ntoreturn":0, "ntoskip":0, "nscanned":19, "keyupdates":0, "numyield":0, "lockstats":{ "timelockedmicros":{ "r":numberlong(380), "w":numberlong(0) }, "timeacquiringmicros":{ "r":numberlong(2), "w":numberlong(4) } }, "nreturned":10, "responselength":36745, "millis":0, "ts": isodate("2014-03-24t09:55:20.988 z"), "client":"127.0.0.1", "allusers":[ { "user":"sytrue-user", "usersource":"sytrue" } ], "user":"sytrue-user@sytrue" } - i notice when retrieve individual record id slow (findone method of pagingandsortingrepository).
- i have mention did not have problem on windows server.
- please explain what's going on.
update: noticed while results returned cpu consumed java code , not mongo server process. problem either in mongodb-java driver or spring-data-mongodb
found problem chatting mongodb developer: running spring framework 4.0.0 , slowness related issue:
https://jira.spring.io/browse/spr-11335
updating spring version 4.0.1 (or above) solves problem.
this stack trace call problem:
@ java.lang.class.getgenericsignature(native method) @ java.lang.class.getgenericinterfaces(class.java:813) @ org.springframework.core.serializabletypewrapper$2.gettype(serializabletypewrapper.java:106) @ org.springframework.core.serializabletypewrapper$typeproxyinvocationhandler.invoke(serializabletypewrapper.java:220) @ com.sun.proxy.$proxy5.equals(unknown source) @ org.springframework.util.objectutils.nullsafeequals(objectutils.java:252) @ org.springframework.core.resolvabletype.equals(resolvabletype.java:682) @ org.springframework.util.objectutils.nullsafeequals(objectutils.java:252) @ org.springframework.core.resolvabletype.variableresolversourceequals(resolvabletype.java:719) @ org.springframework.core.resolvabletype.equals(resolvabletype.java:683) @ org.springframework.util.concurrentreferencehashmap$segment.findinchain(concurrentreferencehashmap.java:571) @ org.springframework.util.concurrentreferencehashmap$segment.getreference(concurrentreferencehashmap.java:439) @ org.springframework.util.concurrentreferencehashmap.getreference(concurrentreferencehashmap.java:238) @ org.springframework.util.concurrentreferencehashmap.get(concurrentreferencehashmap.java:217) @ org.springframework.core.resolvabletype.fortype(resolvabletype.java:1018) @ org.springframework.core.resolvabletype.fortype(resolvabletype.java:998) @ org.springframework.core.resolvabletype.getgenerics(resolvabletype.java:489) @ org.springframework.core.resolvabletype.getgeneric(resolvabletype.java:451) @ org.springframework.core.convert.typedescriptor.getelementtypedescriptor(typedescriptor.java:305) @ org.springframework.core.convert.typedescriptor.equals(typedescriptor.java:456) @ org.springframework.util.objectutils.nullsafeequals(objectutils.java:252) @ org.springframework.core.convert.support.genericconversionservice$convertercachekey.equals(genericconversionservice.java:436) @ java.util.concurrent.concurrenthashmap.get(concurrenthashmap.java:996) @ org.springframework.core.convert.support.genericconversionservice.getconverter(genericconversionservice.java:235) @ org.springframework.core.convert.support.genericconversionservice.convert(genericconversionservice.java:176) @ org.springframework.core.convert.support.genericconversionservice.convert(genericconversionservice.java:162) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.getpotentiallyconvertedsimpleread(mappingmongoconverter.java:715) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.readcollectionorarray(mappingmongoconverter.java:773) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.readvalue(mappingmongoconverter.java:1048) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.access$100(mappingmongoconverter.java:77) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter$mongodbpropertyvalueprovider.getpropertyvalue(mappingmongoconverter.java:999) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.getvalueinternal(mappingmongoconverter.java:755) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter$1.dowithpersistentproperty(mappingmongoconverter.java:256) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter$1.dowithpersistentproperty(mappingmongoconverter.java:249) @ org.springframework.data.mapping.model.basicpersistententity.dowithproperties(basicpersistententity.java:261) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.read(mappingmongoconverter.java:249) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.read(mappingmongoconverter.java:223) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.read(mappingmongoconverter.java:187) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.read(mappingmongoconverter.java:183) @ org.springframework.data.mongodb.core.convert.mappingmongoconverter.read(mappingmongoconverter.java:77) @ org.springframework.data.mongodb.core.mongotemplate$readdbobjectcallback.dowith(mongotemplate.java:1966) @ org.springframework.data.mongodb.core.mongotemplate.executefindmultiinternal(mongotemplate.java:1662) @ org.springframework.data.mongodb.core.mongotemplate.dofind(mongotemplate.java:1487) @ org.springframework.data.mongodb.core.mongotemplate.dofind(mongotemplate.java:1471) @ org.springframework.data.mongodb.core.mongotemplate.find(mongotemplate.java:527) @ org.springframework.data.mongodb.core.mongotemplate.find(mongotemplate.java:518) @ com.sytrue.record.persistence.repo.impl.recordrepositoryimpl.findasmapids(recordrepositoryimpl.java:139)
Comments
Post a Comment