User-agent: *
Disallow: /admin/ # admin dir
Disallow: /cron/ #cron jobs
Disallow: /inc/ # includes
Disallow: /libs/ # libraries