Change line 1:
t=linspace(0,1,256);
fc=60;
fm=15;
ac=1;
am=1;
wc=2*pi*fc;
wm=2*pi*fm;
mt=am*cos(wm*t);
m_ac=ac*cos(wc*t);
%mssb=mt*cos(2*pi*(fc-fm)t)-.*cos(2*pi(fc+fm)*t;
%mssb=ac*sin(wc)+(m*ac/2).*cos(wc-wm)-(m*ac/2).*cos(wc+wm);
xssb=mt.*cos(wc*t)+(mt.*sin(wc*t));
wc=2*pi*fc;
wm=2*pi*fm;
mt=am*cos(wm*t);
sc=ac*cos(wc*t);
xam=am*cos(wm*t).*cos(wc*t);
x_vsb=(xssb+xam)/2;
n=256;
fs=3*fc;
%vm=cos(wm*t);
mt_fssb=fft(xssb,n);
%fft_vm=abs(fft_vm(1:length(fft_vm)/2+1));
fssb=[-(length(mt_fssb))/2:(length(mt_fssb))/2-1]*fs/length(mt_fssb);
mt_amdsb=fft(xam,n);
fdsb=[-(length(mt_amdsb))/2:(length(mt_amdsb))/2-1]*fs/length(mt_amdsb);
mt_vsb=fft(x_vsb,n);
f_vsb=[-(length(mt_vsb))/2:(length(mt_vsb))/2-1]*fs/length(mt_vsb);
fo=45;
wo=2*pi*fo;
w=2*pi*f_vsb;
n1=8;
m=2*n1;
wf=w/wo;
wf2=wf.^m;
wf3=sqrt(1+wf2);
hw=(1./wf3);
%hw=1/(sqrt(1+(w1/wo)^(2*n)));
dm_vsb=mt_vsb.*hw;
subplot(2,2,1);
plot(t,abs(x_vsb));
title('vsb domain w');
xlabel('Time');
ylabel('Amplitude');
length (f_vsb);
length (mt_vsb);
length (abs(x_vsb));
subplot(2,2,2);
plot3(f_vsb,mt_vsb,abs(x_vsb));
title('vsb domain f');
xlabel('Frequency');
ylabel('Amplitude');
subplot(2,2,3);
plot(f_vsb,hw);
title('Filter');
xlabel('Frequency');
ylabel('Amplitude');
subplot(2,2,4);
plot(f_vsb,abs(mt_vsb));
title('demodulation');
xlabel('Frequency');
ylabel('Amplitude');
