بسته rotating جعبه‌ای می‌سازد که قابلیت شکسته شدن بین صفحات را ندارد. اما بسته lscape محیط landscape را فراهم می‌آورد که می‌تواند بین صفحات بشکند و با longtable و supertabular سازگار است. بنابراین کافی است به جای بسته rotating و محیط sidewaystable از بسته lscape و محیط landscape استفاده کنید.
کد شما باید به صورت زیر باشد:
\documentclass[a4paper,12pt]{report}
\usepackage{amssymb,amsmath,amsthm}
\usepackage{lscape}
\usepackage{supertabular}
\usepackage{array}
\newcolumntype{C}[1]{>{\centering\arraybackslash}m{#1}}
\usepackage{xepersian}
\settextfont[Scale=1.1]{XB Yas}
\setdigitfont[Scale=1.1]{Yas}
\begin{document}
\begin{landscape}
\small
\tablefirsthead{\hline
$1$&$0.9$&$0.8$&$0.7$&$0.6$&$0.5$&$0.4$&$0.3$&$0.2$&$0.1$&$0$& &$k_1/k_2$ \\ \hline}
\tablehead{\hline
$1$&$0.9$&$0.8$&$0.7$&$0.6$&$0.5$&$0.4$&$0.3$&$0.2$&$0.1$&$0$& &$k_1/k_2$ \\ \hline}
\tabletail{\hline}
\tablelasttail{\hline}
\tablecaption{}\label{tab111}
\renewcommand{\arraystretch}{1.6}
%\begin{supertabular}{c|c|c|c|c|c|c|c|c|c|c|c|c}
\begin{supertabular}{C{1cm}*{12}{|C{1cm}}}
$1$&$0.999$&$0.999$&$0.999$&$0.999$&$0.999$&$0.998$&$0.998$&$0.998$ &$0.998$&$0.998$&$3$&$3$ \\ \hline
$0.978$&$0.978$&$0.978$&$0.978$&$0.978$&$0.977$&$0.977$ &$0.977$&$0.977$&$0.977$&$0.977$&$2$&$3$ \\ \hline
$0.842$&$0.842$&$0.842$&$0.842$&$0.842$&$0.842$&$0.841$&$0.841$&$0.841$ &$0.841$&$0.842$&$1$&$3$ \\ \hline
$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$ &$0.499$&$0.499$&$0$&$3$ \\ \hline
$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$ &$0.158$&$0.158$&$-1$&$3$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$ &$0.022$&$0.022$&$-2$&$3$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$3$ \\ \hline
$0.978$&$0.978$&$0.978$&$0.978$&$0.978$&$0.977$&$0.977$ &$0.977$&$0.977$&$0.977$&$0.977$&$3$&$2$ \\ \hline
$0.977$&$0.969$&$0.965$&$0.963$&$0.961$&$0.960$&$0.958$ &$0.957$&$0.957$&$0.956$&$0.956$&$2$&$2$ \\ \hline
$0.842$&$0.842$&$0.840$&$0.838$&$0.835$&$0.832$&$0.830$&$0.828$&$0.826$&$0.824$&$0.823$&$1$&$2$ \\ \hline
$0.500$&$0.500$&$0.500$&$0.500$&$0.499$&$0.498$&$0.497$&$0.495$&$0.493$&$0.491$&$0.489$&$0$&$2$ \\ \hline
$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.157$&$0.157$&$0.156$&$0.154$&$-1$&$2$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.021$&$0.021$&$-2$&$2$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$2$ \\ \hline
$0.842$&$0.842$&$0.842$&$0.842$&$0.842$&$0.842$&$0.842$&$0.841$&$0.841$ &$0.841$&$0.841$&$3$&$1$ \\ \hline
$0.842$&$0.842$&$0.840$&$0.838$&$0.835$&$0.833$&$0.830$&$0.828$&$0.826$&$0.824$&$0.823$&$2$&$1$ \\ \hline
$0.836$&$0.799$&$0.781$&$0.767$&$0.756$&$0.746$&$0.737$&$0.729$&$0.721$&$0.714$&$0.708$&$1$&$1$ \\ \hline
$0.500$&$0.499$&$0.494$&$0.487$&$0.478$&$0.469$&$0.459$&$0.450$&$0.440$&$0.430$&$0.421$&$0$&$1$ \\ \hline
$0.158$&$0.158$&$0.158$&$0.158$&$0.156$&$0.154$&$0.151$&$0.148$&$0.143$&$0.138$&$0.133$&$-1$&$1$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.021$&$0.021$&$0.020$&$0.019$&$0.018$&$-2$&$1$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$1$ \\ \hline
$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$0.500$&$3$&$0$ \\ \hline
$0.500$&$0.500$&$0.500$&$0.500$&$0.499$&$0.498$&$0.497$&$0.495$&$0.493$&$0.491$&$0.489$&$2$&$0$ \\ \hline
$0.500$&$0.499$&$0.494$&$0.487$&$0.478$&$0.469$&$0.459$&$0.450$&$0.440$&$0.430$&$0.421$&$1$&$0$ \\ \hline
$0.490$&$0.428$&$0.397$&$0.373$&$0.352$&$0.333$&$0.315$&$0.298$&$0.282$&$0.266$&$0.250$&$0$&$0$ \\ \hline
$0.158$&$0.157$&$0.152$&$0.145$&$0.136$&$0.127$&$0.117$&$0.108$&$0.098$&$0.089$&$0.079$&$-1$&$0$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.021$&$0.021$&$0.020$&$0.018$&$0.017$&$0.015$&$0.013$&$0.011$&$-2$&$0$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$0$ \\ \hline
$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.159$&$0.159$&$0.159$&$3$&$-1$ \\ \hline
$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.158$&$0.157$&$0.156$&$0.155$&$2$&$-1$ \\ \hline
$0.158$&$0.158$&$0.158$&$0.157$&$0.156$&$0.154$&$0.152$&$0.148$&$0.144$&$0.139$&$0.134$&$1$&$-1$ \\ \hline
$0.158$&$0.157$&$0.152$&$0.145$&$0.136$&$0.127$&$0.117$&$0.108$&$0.098$&$0.089$&$0.079$&$0$&$-1$ \\ \hline
$0.152$&$0.115$&$0.097$&$0.083$&$0.072$&$0.062$&$0.053$&$0.045$&$0.038$&$0.031$&$0.025$&$-1$&$-1$ \\ \hline
$0.022$&$0.021$&$0.020$&$0.017$&$0.015$&$0.012$&$0.010$&$0.008$&$0.006$&$0.005$&$0.003$&$-2$&$-1$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$-1$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.023$&$0.023$&$0.023$&$0.023$&$0.023$&$3$&$-2$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.023$&$0.023$&$0.023$&$0.022$&$0.022$&$2$&$-2$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.022$&$0.021$&$0.020$&$0.019$&$1$&$-2$ \\ \hline
$0.022$&$0.022$&$0.022$&$0.022$&$0.021$&$0.020$&$0.019$&$0.017$&$0.016$&$0.013$&$0.011$&$0$&$-2$ \\ \hline
$0.022$&$0.021$&$0.020$&$0.018$&$0.015$&$0.013$&$0.011$&$0.009$&$0.007$&$0.005$&$0.004$&$-1$&$-2$ \\ \hline
$0.020$&$0.012$&$0.009$&$0.007$&$0.005$&$0.004$&$0.003$&$0.002$&$0.001$&$1$&$0$&$-2$&$-2$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$-2$ \\ \hline
$0$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$3$&$-3$ \\ \hline
$0$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$2$&$-3$ \\ \hline
$0$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$1$&$-3$ \\ \hline
$0$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0$&$-3$ \\ \hline
$0$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0.001$&$0$&$0$&$0$&$-1$&$-3$ \\ \hline
$0$&$0.001$&$0.001$&$0.001$&$0.001$&$0$&$0$&$0$&$0$&$0$&$0$&$-2$&$-3$ \\ \hline
$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$0$&$-3$&$-3$ \\ \hline
\end{supertabular}
\end{landscape}
\end{document}
چند نکته:
شما تعداد زیادی بسته را بعد از بسته xepersian آورده‌اید که این کار اشتباه است. بسته xepersian را باید به عنوان آخرین بسته فراخوانی کنید.
چون محتوای جدول در هر سطر متفاوت است، اگر نوع هر ستون را به سادگی وسط‌چین (c) تعریف کنیم، وقتی جدول شکسته می‌شود و به صفحه بعد می‌رود ظاهر آن با آنچه در صفحه قبلی وجود دارد متفاوت می‌شود. برای جلوگیری از این اتفاق به کمک بسته array عرض هر ستون را مقداری مشخص و ثابت در نظر می‌گیریم. برای این کار قاعده جدید زیر را تعریف کرده و در ابتدای محیط جدول از آن استفاده می‌کنیم.
\newcolumntype{C}[1]{>{\centering\arraybackslash}m{#1}}
بسته lscape روی خروجی pdf تأثیری نمی‌گذارد و آن را نمی‌چرخاند. اگر چنین چیزی می‌خواهید کافی است به جای بسته lscape از بسته pdflscape استفاده کنید و نتیجه را ببینید.
